aboutsummaryrefslogtreecommitdiffhomepage
path: root/src/Wallabag/ImportBundle/Import
diff options
context:
space:
mode:
Diffstat (limited to 'src/Wallabag/ImportBundle/Import')
-rw-r--r--src/Wallabag/ImportBundle/Import/AbstractImport.php24
-rw-r--r--src/Wallabag/ImportBundle/Import/BrowserImport.php17
-rw-r--r--src/Wallabag/ImportBundle/Import/ChromeImport.php5
-rw-r--r--src/Wallabag/ImportBundle/Import/FirefoxImport.php5
-rw-r--r--src/Wallabag/ImportBundle/Import/InstapaperImport.php3
-rw-r--r--src/Wallabag/ImportBundle/Import/PinboardImport.php141
-rw-r--r--src/Wallabag/ImportBundle/Import/PocketImport.php9
-rw-r--r--src/Wallabag/ImportBundle/Import/ReadabilityImport.php3
-rw-r--r--src/Wallabag/ImportBundle/Import/WallabagV1Import.php2
-rw-r--r--src/Wallabag/ImportBundle/Import/WallabagV2Import.php3
10 files changed, 191 insertions, 21 deletions
diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php
index 764b390a..1d4a6e27 100644
--- a/src/Wallabag/ImportBundle/Import/AbstractImport.php
+++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php
@@ -10,12 +10,15 @@ use Wallabag\CoreBundle\Entity\Entry;
10use Wallabag\CoreBundle\Entity\Tag; 10use Wallabag\CoreBundle\Entity\Tag;
11use Wallabag\UserBundle\Entity\User; 11use Wallabag\UserBundle\Entity\User;
12use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface; 12use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface;
13use Symfony\Component\EventDispatcher\EventDispatcherInterface;
14use Wallabag\CoreBundle\Event\EntrySavedEvent;
13 15
14abstract class AbstractImport implements ImportInterface 16abstract class AbstractImport implements ImportInterface
15{ 17{
16 protected $em; 18 protected $em;
17 protected $logger; 19 protected $logger;
18 protected $contentProxy; 20 protected $contentProxy;
21 protected $eventDispatcher;
19 protected $producer; 22 protected $producer;
20 protected $user; 23 protected $user;
21 protected $markAsRead; 24 protected $markAsRead;
@@ -23,11 +26,12 @@ abstract class AbstractImport implements ImportInterface
23 protected $importedEntries = 0; 26 protected $importedEntries = 0;
24 protected $queuedEntries = 0; 27 protected $queuedEntries = 0;
25 28
26 public function __construct(EntityManager $em, ContentProxy $contentProxy) 29 public function __construct(EntityManager $em, ContentProxy $contentProxy, EventDispatcherInterface $eventDispatcher)
27 { 30 {
28 $this->em = $em; 31 $this->em = $em;
29 $this->logger = new NullLogger(); 32 $this->logger = new NullLogger();
30 $this->contentProxy = $contentProxy; 33 $this->contentProxy = $contentProxy;
34 $this->eventDispatcher = $eventDispatcher;
31 } 35 }
32 36
33 public function setLogger(LoggerInterface $logger) 37 public function setLogger(LoggerInterface $logger)
@@ -104,6 +108,7 @@ abstract class AbstractImport implements ImportInterface
104 protected function parseEntries($entries) 108 protected function parseEntries($entries)
105 { 109 {
106 $i = 1; 110 $i = 1;
111 $entryToBeFlushed = [];
107 112
108 foreach ($entries as $importedEntry) { 113 foreach ($entries as $importedEntry) {
109 if ($this->markAsRead) { 114 if ($this->markAsRead) {
@@ -116,10 +121,21 @@ abstract class AbstractImport implements ImportInterface
116 continue; 121 continue;
117 } 122 }
118 123
124 // store each entry to be flushed so we can trigger the entry.saved event for each of them
125 // entry.saved needs the entry to be persisted in db because it needs it id to generate
126 // images (at least)
127 $entryToBeFlushed[] = $entry;
128
119 // flush every 20 entries 129 // flush every 20 entries
120 if (($i % 20) === 0) { 130 if (($i % 20) === 0) {
121 $this->em->flush(); 131 $this->em->flush();
122 132
133 foreach ($entryToBeFlushed as $entry) {
134 $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry));
135 }
136
137 $entryToBeFlushed = [];
138
123 // clear only affected entities 139 // clear only affected entities
124 $this->em->clear(Entry::class); 140 $this->em->clear(Entry::class);
125 $this->em->clear(Tag::class); 141 $this->em->clear(Tag::class);
@@ -128,6 +144,12 @@ abstract class AbstractImport implements ImportInterface
128 } 144 }
129 145
130 $this->em->flush(); 146 $this->em->flush();
147
148 if (!empty($entryToBeFlushed)) {
149 foreach ($entryToBeFlushed as $entry) {
150 $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry));
151 }
152 }
131 } 153 }
132 154
133 /** 155 /**
diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php
index 2ca1683b..8bf7d92e 100644
--- a/src/Wallabag/ImportBundle/Import/BrowserImport.php
+++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php
@@ -5,6 +5,7 @@ namespace Wallabag\ImportBundle\Import;
5use Wallabag\CoreBundle\Entity\Entry; 5use Wallabag\CoreBundle\Entity\Entry;
6use Wallabag\UserBundle\Entity\User; 6use Wallabag\UserBundle\Entity\User;
7use Wallabag\CoreBundle\Helper\ContentProxy; 7use Wallabag\CoreBundle\Helper\ContentProxy;
8use Wallabag\CoreBundle\Event\EntrySavedEvent;
8 9
9abstract class BrowserImport extends AbstractImport 10abstract class BrowserImport extends AbstractImport
10{ 11{
@@ -81,6 +82,7 @@ abstract class BrowserImport extends AbstractImport
81 protected function parseEntries($entries) 82 protected function parseEntries($entries)
82 { 83 {
83 $i = 1; 84 $i = 1;
85 $entryToBeFlushed = [];
84 86
85 foreach ($entries as $importedEntry) { 87 foreach ($entries as $importedEntry) {
86 if ((array) $importedEntry !== $importedEntry) { 88 if ((array) $importedEntry !== $importedEntry) {
@@ -93,14 +95,29 @@ abstract class BrowserImport extends AbstractImport
93 continue; 95 continue;
94 } 96 }
95 97
98 // @see AbstractImport
99 $entryToBeFlushed[] = $entry;
100
96 // flush every 20 entries 101 // flush every 20 entries
97 if (($i % 20) === 0) { 102 if (($i % 20) === 0) {
98 $this->em->flush(); 103 $this->em->flush();
104
105 foreach ($entryToBeFlushed as $entry) {
106 $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry));
107 }
108
109 $entryToBeFlushed = [];
99 } 110 }
100 ++$i; 111 ++$i;
101 } 112 }
102 113
103 $this->em->flush(); 114 $this->em->flush();
115
116 if (!empty($entryToBeFlushed)) {
117 foreach ($entryToBeFlushed as $entry) {
118 $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry));
119 }
120 }
104 } 121 }
105 122
106 /** 123 /**
diff --git a/src/Wallabag/ImportBundle/Import/ChromeImport.php b/src/Wallabag/ImportBundle/Import/ChromeImport.php
index d7620bcb..2667890f 100644
--- a/src/Wallabag/ImportBundle/Import/ChromeImport.php
+++ b/src/Wallabag/ImportBundle/Import/ChromeImport.php
@@ -37,9 +37,10 @@ class ChromeImport extends BrowserImport
37 { 37 {
38 $data = [ 38 $data = [
39 'title' => $entry['name'], 39 'title' => $entry['name'],
40 'html' => '', 40 'html' => false,
41 'url' => $entry['url'], 41 'url' => $entry['url'],
42 'is_archived' => $this->markAsRead, 42 'is_archived' => (int) $this->markAsRead,
43 'is_starred' => false,
43 'tags' => '', 44 'tags' => '',
44 'created_at' => substr($entry['date_added'], 0, 10), 45 'created_at' => substr($entry['date_added'], 0, 10),
45 ]; 46 ];
diff --git a/src/Wallabag/ImportBundle/Import/FirefoxImport.php b/src/Wallabag/ImportBundle/Import/FirefoxImport.php
index e010f5a4..c50c69b3 100644
--- a/src/Wallabag/ImportBundle/Import/FirefoxImport.php
+++ b/src/Wallabag/ImportBundle/Import/FirefoxImport.php
@@ -37,9 +37,10 @@ class FirefoxImport extends BrowserImport
37 { 37 {
38 $data = [ 38 $data = [
39 'title' => $entry['title'], 39 'title' => $entry['title'],
40 'html' => '', 40 'html' => false,
41 'url' => $entry['uri'], 41 'url' => $entry['uri'],
42 'is_archived' => $this->markAsRead, 42 'is_archived' => (int) $this->markAsRead,
43 'is_starred' => false,
43 'tags' => '', 44 'tags' => '',
44 'created_at' => substr($entry['dateAdded'], 0, 10), 45 'created_at' => substr($entry['dateAdded'], 0, 10),
45 ]; 46 ];
diff --git a/src/Wallabag/ImportBundle/Import/InstapaperImport.php b/src/Wallabag/ImportBundle/Import/InstapaperImport.php
index cf4c785c..70a53f1a 100644
--- a/src/Wallabag/ImportBundle/Import/InstapaperImport.php
+++ b/src/Wallabag/ImportBundle/Import/InstapaperImport.php
@@ -74,8 +74,7 @@ class InstapaperImport extends AbstractImport
74 'status' => $data[3], 74 'status' => $data[3],
75 'is_archived' => $data[3] === 'Archive' || $data[3] === 'Starred', 75 'is_archived' => $data[3] === 'Archive' || $data[3] === 'Starred',
76 'is_starred' => $data[3] === 'Starred', 76 'is_starred' => $data[3] === 'Starred',
77 'content_type' => '', 77 'html' => false,
78 'language' => '',
79 ]; 78 ];
80 } 79 }
81 fclose($handle); 80 fclose($handle);
diff --git a/src/Wallabag/ImportBundle/Import/PinboardImport.php b/src/Wallabag/ImportBundle/Import/PinboardImport.php
new file mode 100644
index 00000000..d9865534
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Import/PinboardImport.php
@@ -0,0 +1,141 @@
1<?php
2
3namespace Wallabag\ImportBundle\Import;
4
5use Wallabag\CoreBundle\Entity\Entry;
6
7class PinboardImport extends AbstractImport
8{
9 private $filepath;
10
11 /**
12 * {@inheritdoc}
13 */
14 public function getName()
15 {
16 return 'Pinboard';
17 }
18
19 /**
20 * {@inheritdoc}
21 */
22 public function getUrl()
23 {
24 return 'import_pinboard';
25 }
26
27 /**
28 * {@inheritdoc}
29 */
30 public function getDescription()
31 {
32 return 'import.pinboard.description';
33 }
34
35 /**
36 * Set file path to the json file.
37 *
38 * @param string $filepath
39 */
40 public function setFilepath($filepath)
41 {
42 $this->filepath = $filepath;
43
44 return $this;
45 }
46
47 /**
48 * {@inheritdoc}
49 */
50 public function import()
51 {
52 if (!$this->user) {
53 $this->logger->error('PinboardImport: user is not defined');
54
55 return false;
56 }
57
58 if (!file_exists($this->filepath) || !is_readable($this->filepath)) {
59 $this->logger->error('PinboardImport: unable to read file', ['filepath' => $this->filepath]);
60
61 return false;
62 }
63
64 $data = json_decode(file_get_contents($this->filepath), true);
65
66 if (empty($data)) {
67 $this->logger->error('PinboardImport: no entries in imported file');
68
69 return false;
70 }
71
72 if ($this->producer) {
73 $this->parseEntriesForProducer($data);
74
75 return true;
76 }
77
78 $this->parseEntries($data);
79
80 return true;
81 }
82
83 /**
84 * {@inheritdoc}
85 */
86 public function parseEntry(array $importedEntry)
87 {
88 $existingEntry = $this->em
89 ->getRepository('WallabagCoreBundle:Entry')
90 ->findByUrlAndUserId($importedEntry['href'], $this->user->getId());
91
92 if (false !== $existingEntry) {
93 ++$this->skippedEntries;
94
95 return;
96 }
97
98 $data = [
99 'title' => $importedEntry['description'],
100 'url' => $importedEntry['href'],
101 'is_archived' => ('no' === $importedEntry['toread']) || $this->markAsRead,
102 'is_starred' => false,
103 'created_at' => $importedEntry['time'],
104 'tags' => explode(' ', $importedEntry['tags']),
105 ];
106
107 $entry = new Entry($this->user);
108 $entry->setUrl($data['url']);
109 $entry->setTitle($data['title']);
110
111 // update entry with content (in case fetching failed, the given entry will be return)
112 $entry = $this->fetchContent($entry, $data['url'], $data);
113
114 if (!empty($data['tags'])) {
115 $this->contentProxy->assignTagsToEntry(
116 $entry,
117 $data['tags'],
118 $this->em->getUnitOfWork()->getScheduledEntityInsertions()
119 );
120 }
121
122 $entry->setArchived($data['is_archived']);
123 $entry->setStarred($data['is_starred']);
124 $entry->setCreatedAt(new \DateTime($data['created_at']));
125
126 $this->em->persist($entry);
127 ++$this->importedEntries;
128
129 return $entry;
130 }
131
132 /**
133 * {@inheritdoc}
134 */
135 protected function setEntryAsRead(array $importedEntry)
136 {
137 $importedEntry['toread'] = 'no';
138
139 return $importedEntry;
140 }
141}
diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php
index 327e2500..33093480 100644
--- a/src/Wallabag/ImportBundle/Import/PocketImport.php
+++ b/src/Wallabag/ImportBundle/Import/PocketImport.php
@@ -2,8 +2,6 @@
2 2
3namespace Wallabag\ImportBundle\Import; 3namespace Wallabag\ImportBundle\Import;
4 4
5use Psr\Log\NullLogger;
6use Doctrine\ORM\EntityManager;
7use GuzzleHttp\Client; 5use GuzzleHttp\Client;
8use GuzzleHttp\Exception\RequestException; 6use GuzzleHttp\Exception\RequestException;
9use Wallabag\CoreBundle\Entity\Entry; 7use Wallabag\CoreBundle\Entity\Entry;
@@ -16,13 +14,6 @@ class PocketImport extends AbstractImport
16 14
17 const NB_ELEMENTS = 5000; 15 const NB_ELEMENTS = 5000;
18 16
19 public function __construct(EntityManager $em, ContentProxy $contentProxy)
20 {
21 $this->em = $em;
22 $this->contentProxy = $contentProxy;
23 $this->logger = new NullLogger();
24 }
25
26 /** 17 /**
27 * Only used for test purpose. 18 * Only used for test purpose.
28 * 19 *
diff --git a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
index b8c0f777..de320d23 100644
--- a/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
+++ b/src/Wallabag/ImportBundle/Import/ReadabilityImport.php
@@ -98,11 +98,10 @@ class ReadabilityImport extends AbstractImport
98 $data = [ 98 $data = [
99 'title' => $importedEntry['article__title'], 99 'title' => $importedEntry['article__title'],
100 'url' => $importedEntry['article__url'], 100 'url' => $importedEntry['article__url'],
101 'content_type' => '',
102 'language' => '',
103 'is_archived' => $importedEntry['archive'] || $this->markAsRead, 101 'is_archived' => $importedEntry['archive'] || $this->markAsRead,
104 'is_starred' => $importedEntry['favorite'], 102 'is_starred' => $importedEntry['favorite'],
105 'created_at' => $importedEntry['date_added'], 103 'created_at' => $importedEntry['date_added'],
104 'html' => false,
106 ]; 105 ];
107 106
108 $entry = new Entry($this->user); 107 $entry = new Entry($this->user);
diff --git a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php
index 4f001062..59e3ce02 100644
--- a/src/Wallabag/ImportBundle/Import/WallabagV1Import.php
+++ b/src/Wallabag/ImportBundle/Import/WallabagV1Import.php
@@ -37,8 +37,6 @@ class WallabagV1Import extends WallabagImport
37 'title' => $entry['title'], 37 'title' => $entry['title'],
38 'html' => $entry['content'], 38 'html' => $entry['content'],
39 'url' => $entry['url'], 39 'url' => $entry['url'],
40 'content_type' => '',
41 'language' => '',
42 'is_archived' => $entry['is_read'] || $this->markAsRead, 40 'is_archived' => $entry['is_read'] || $this->markAsRead,
43 'is_starred' => $entry['is_fav'], 41 'is_starred' => $entry['is_fav'],
44 'tags' => '', 42 'tags' => '',
diff --git a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php
index 37c8ca14..d2a89d79 100644
--- a/src/Wallabag/ImportBundle/Import/WallabagV2Import.php
+++ b/src/Wallabag/ImportBundle/Import/WallabagV2Import.php
@@ -36,7 +36,8 @@ class WallabagV2Import extends WallabagImport
36 return [ 36 return [
37 'html' => $entry['content'], 37 'html' => $entry['content'],
38 'content_type' => $entry['mimetype'], 38 'content_type' => $entry['mimetype'],
39 'is_archived' => ($entry['is_archived'] || $this->markAsRead), 39 'is_archived' => (int) ($entry['is_archived'] || $this->markAsRead),
40 'is_starred' => false,
40 ] + $entry; 41 ] + $entry;
41 } 42 }
42 43