diff options
author | ArthurHoaro <arthur@hoa.ro> | 2017-10-07 16:40:16 +0200 |
---|---|---|
committer | ArthurHoaro <arthur@hoa.ro> | 2017-10-07 16:40:16 +0200 |
commit | 66e74d50d38a6fea8fc904a1746157633de7cc65 (patch) | |
tree | ad19443a85975b017cabb159b2d3a56732cd1164 /application | |
parent | 78865393a687a4c057109fa51f22934ad078d482 (diff) | |
download | Shaarli-66e74d50d38a6fea8fc904a1746157633de7cc65.tar.gz Shaarli-66e74d50d38a6fea8fc904a1746157633de7cc65.tar.zst Shaarli-66e74d50d38a6fea8fc904a1746157633de7cc65.zip |
Don't write History for link import
With large imports it has a large impact on performances and isn't really useful.
Instead, write an IMPORT event, which let client using the history service resync its DB.
-> 15k link import done in 6 seconds.
Fixes #985
Diffstat (limited to 'application')
-rw-r--r-- | application/History.php | 16 | ||||
-rw-r--r-- | application/NetscapeBookmarkUtils.php | 16 |
2 files changed, 27 insertions, 5 deletions
diff --git a/application/History.php b/application/History.php index 116b9264..5e3b1b72 100644 --- a/application/History.php +++ b/application/History.php | |||
@@ -16,6 +16,7 @@ | |||
16 | * - UPDATED: link updated | 16 | * - UPDATED: link updated |
17 | * - DELETED: link deleted | 17 | * - DELETED: link deleted |
18 | * - SETTINGS: the settings have been updated through the UI. | 18 | * - SETTINGS: the settings have been updated through the UI. |
19 | * - IMPORT: bulk links import | ||
19 | * | 20 | * |
20 | * Note: new events are put at the beginning of the file and history array. | 21 | * Note: new events are put at the beginning of the file and history array. |
21 | */ | 22 | */ |
@@ -42,6 +43,11 @@ class History | |||
42 | const SETTINGS = 'SETTINGS'; | 43 | const SETTINGS = 'SETTINGS'; |
43 | 44 | ||
44 | /** | 45 | /** |
46 | * @var string Action key: a bulk import has been processed. | ||
47 | */ | ||
48 | const IMPORT = 'IMPORT'; | ||
49 | |||
50 | /** | ||
45 | * @var string History file path. | 51 | * @var string History file path. |
46 | */ | 52 | */ |
47 | protected $historyFilePath; | 53 | protected $historyFilePath; |
@@ -122,6 +128,16 @@ class History | |||
122 | } | 128 | } |
123 | 129 | ||
124 | /** | 130 | /** |
131 | * Add Event: bulk import. | ||
132 | * | ||
133 | * Note: we don't store links add/update one by one since it can have a huge impact on performances. | ||
134 | */ | ||
135 | public function importLinks() | ||
136 | { | ||
137 | $this->addEvent(self::IMPORT); | ||
138 | } | ||
139 | |||
140 | /** | ||
125 | * Save a new event and write it in the history file. | 141 | * Save a new event and write it in the history file. |
126 | * | 142 | * |
127 | * @param string $status Event key, should be defined as constant. | 143 | * @param string $status Event key, should be defined as constant. |
diff --git a/application/NetscapeBookmarkUtils.php b/application/NetscapeBookmarkUtils.php index 2a10ff22..31796367 100644 --- a/application/NetscapeBookmarkUtils.php +++ b/application/NetscapeBookmarkUtils.php | |||
@@ -66,6 +66,7 @@ class NetscapeBookmarkUtils | |||
66 | * @param int $importCount how many links were imported | 66 | * @param int $importCount how many links were imported |
67 | * @param int $overwriteCount how many links were overwritten | 67 | * @param int $overwriteCount how many links were overwritten |
68 | * @param int $skipCount how many links were skipped | 68 | * @param int $skipCount how many links were skipped |
69 | * @param int $duration how many seconds did the import take | ||
69 | * | 70 | * |
70 | * @return string Summary of the bookmark import status | 71 | * @return string Summary of the bookmark import status |
71 | */ | 72 | */ |
@@ -74,14 +75,16 @@ class NetscapeBookmarkUtils | |||
74 | $filesize, | 75 | $filesize, |
75 | $importCount=0, | 76 | $importCount=0, |
76 | $overwriteCount=0, | 77 | $overwriteCount=0, |
77 | $skipCount=0 | 78 | $skipCount=0, |
79 | $duration=0 | ||
78 | ) | 80 | ) |
79 | { | 81 | { |
80 | $status = 'File '.$filename.' ('.$filesize.' bytes) '; | 82 | $status = 'File '.$filename.' ('.$filesize.' bytes) '; |
81 | if ($importCount == 0 && $overwriteCount == 0 && $skipCount == 0) { | 83 | if ($importCount == 0 && $overwriteCount == 0 && $skipCount == 0) { |
82 | $status .= 'has an unknown file format. Nothing was imported.'; | 84 | $status .= 'has an unknown file format. Nothing was imported.'; |
83 | } else { | 85 | } else { |
84 | $status .= 'was successfully processed: '.$importCount.' links imported, '; | 86 | $status .= 'was successfully processed in '. $duration .' seconds: '; |
87 | $status .= $importCount.' links imported, '; | ||
85 | $status .= $overwriteCount.' links overwritten, '; | 88 | $status .= $overwriteCount.' links overwritten, '; |
86 | $status .= $skipCount.' links skipped.'; | 89 | $status .= $skipCount.' links skipped.'; |
87 | } | 90 | } |
@@ -101,6 +104,7 @@ class NetscapeBookmarkUtils | |||
101 | */ | 104 | */ |
102 | public static function import($post, $files, $linkDb, $conf, $history) | 105 | public static function import($post, $files, $linkDb, $conf, $history) |
103 | { | 106 | { |
107 | $start = time(); | ||
104 | $filename = $files['filetoupload']['name']; | 108 | $filename = $files['filetoupload']['name']; |
105 | $filesize = $files['filetoupload']['size']; | 109 | $filesize = $files['filetoupload']['size']; |
106 | $data = file_get_contents($files['filetoupload']['tmp_name']); | 110 | $data = file_get_contents($files['filetoupload']['tmp_name']); |
@@ -184,7 +188,6 @@ class NetscapeBookmarkUtils | |||
184 | $linkDb[$existingLink['id']] = $newLink; | 188 | $linkDb[$existingLink['id']] = $newLink; |
185 | $importCount++; | 189 | $importCount++; |
186 | $overwriteCount++; | 190 | $overwriteCount++; |
187 | $history->updateLink($newLink); | ||
188 | continue; | 191 | continue; |
189 | } | 192 | } |
190 | 193 | ||
@@ -196,16 +199,19 @@ class NetscapeBookmarkUtils | |||
196 | $newLink['shorturl'] = link_small_hash($newLink['created'], $newLink['id']); | 199 | $newLink['shorturl'] = link_small_hash($newLink['created'], $newLink['id']); |
197 | $linkDb[$newLink['id']] = $newLink; | 200 | $linkDb[$newLink['id']] = $newLink; |
198 | $importCount++; | 201 | $importCount++; |
199 | $history->addLink($newLink); | ||
200 | } | 202 | } |
201 | 203 | ||
202 | $linkDb->save($conf->get('resource.page_cache')); | 204 | $linkDb->save($conf->get('resource.page_cache')); |
205 | $history->importLinks(); | ||
206 | |||
207 | $duration = time() - $start; | ||
203 | return self::importStatus( | 208 | return self::importStatus( |
204 | $filename, | 209 | $filename, |
205 | $filesize, | 210 | $filesize, |
206 | $importCount, | 211 | $importCount, |
207 | $overwriteCount, | 212 | $overwriteCount, |
208 | $skipCount | 213 | $skipCount, |
214 | $duration | ||
209 | ); | 215 | ); |
210 | } | 216 | } |
211 | } | 217 | } |