diff options
Diffstat (limited to 'src/Wallabag/ImportBundle/Import')
-rw-r--r-- | src/Wallabag/ImportBundle/Import/AbstractImport.php | 24 | ||||
-rw-r--r-- | src/Wallabag/ImportBundle/Import/BrowserImport.php | 17 | ||||
-rw-r--r-- | src/Wallabag/ImportBundle/Import/PinboardImport.php | 143 | ||||
-rw-r--r-- | src/Wallabag/ImportBundle/Import/PocketImport.php | 9 |
4 files changed, 183 insertions, 10 deletions
diff --git a/src/Wallabag/ImportBundle/Import/AbstractImport.php b/src/Wallabag/ImportBundle/Import/AbstractImport.php index 764b390a..1d4a6e27 100644 --- a/src/Wallabag/ImportBundle/Import/AbstractImport.php +++ b/src/Wallabag/ImportBundle/Import/AbstractImport.php | |||
@@ -10,12 +10,15 @@ use Wallabag\CoreBundle\Entity\Entry; | |||
10 | use Wallabag\CoreBundle\Entity\Tag; | 10 | use Wallabag\CoreBundle\Entity\Tag; |
11 | use Wallabag\UserBundle\Entity\User; | 11 | use Wallabag\UserBundle\Entity\User; |
12 | use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface; | 12 | use OldSound\RabbitMqBundle\RabbitMq\ProducerInterface; |
13 | use Symfony\Component\EventDispatcher\EventDispatcherInterface; | ||
14 | use Wallabag\CoreBundle\Event\EntrySavedEvent; | ||
13 | 15 | ||
14 | abstract class AbstractImport implements ImportInterface | 16 | abstract class AbstractImport implements ImportInterface |
15 | { | 17 | { |
16 | protected $em; | 18 | protected $em; |
17 | protected $logger; | 19 | protected $logger; |
18 | protected $contentProxy; | 20 | protected $contentProxy; |
21 | protected $eventDispatcher; | ||
19 | protected $producer; | 22 | protected $producer; |
20 | protected $user; | 23 | protected $user; |
21 | protected $markAsRead; | 24 | protected $markAsRead; |
@@ -23,11 +26,12 @@ abstract class AbstractImport implements ImportInterface | |||
23 | protected $importedEntries = 0; | 26 | protected $importedEntries = 0; |
24 | protected $queuedEntries = 0; | 27 | protected $queuedEntries = 0; |
25 | 28 | ||
26 | public function __construct(EntityManager $em, ContentProxy $contentProxy) | 29 | public function __construct(EntityManager $em, ContentProxy $contentProxy, EventDispatcherInterface $eventDispatcher) |
27 | { | 30 | { |
28 | $this->em = $em; | 31 | $this->em = $em; |
29 | $this->logger = new NullLogger(); | 32 | $this->logger = new NullLogger(); |
30 | $this->contentProxy = $contentProxy; | 33 | $this->contentProxy = $contentProxy; |
34 | $this->eventDispatcher = $eventDispatcher; | ||
31 | } | 35 | } |
32 | 36 | ||
33 | public function setLogger(LoggerInterface $logger) | 37 | public function setLogger(LoggerInterface $logger) |
@@ -104,6 +108,7 @@ abstract class AbstractImport implements ImportInterface | |||
104 | protected function parseEntries($entries) | 108 | protected function parseEntries($entries) |
105 | { | 109 | { |
106 | $i = 1; | 110 | $i = 1; |
111 | $entryToBeFlushed = []; | ||
107 | 112 | ||
108 | foreach ($entries as $importedEntry) { | 113 | foreach ($entries as $importedEntry) { |
109 | if ($this->markAsRead) { | 114 | if ($this->markAsRead) { |
@@ -116,10 +121,21 @@ abstract class AbstractImport implements ImportInterface | |||
116 | continue; | 121 | continue; |
117 | } | 122 | } |
118 | 123 | ||
124 | // store each entry to be flushed so we can trigger the entry.saved event for each of them | ||
125 | // entry.saved needs the entry to be persisted in db because it needs it id to generate | ||
126 | // images (at least) | ||
127 | $entryToBeFlushed[] = $entry; | ||
128 | |||
119 | // flush every 20 entries | 129 | // flush every 20 entries |
120 | if (($i % 20) === 0) { | 130 | if (($i % 20) === 0) { |
121 | $this->em->flush(); | 131 | $this->em->flush(); |
122 | 132 | ||
133 | foreach ($entryToBeFlushed as $entry) { | ||
134 | $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); | ||
135 | } | ||
136 | |||
137 | $entryToBeFlushed = []; | ||
138 | |||
123 | // clear only affected entities | 139 | // clear only affected entities |
124 | $this->em->clear(Entry::class); | 140 | $this->em->clear(Entry::class); |
125 | $this->em->clear(Tag::class); | 141 | $this->em->clear(Tag::class); |
@@ -128,6 +144,12 @@ abstract class AbstractImport implements ImportInterface | |||
128 | } | 144 | } |
129 | 145 | ||
130 | $this->em->flush(); | 146 | $this->em->flush(); |
147 | |||
148 | if (!empty($entryToBeFlushed)) { | ||
149 | foreach ($entryToBeFlushed as $entry) { | ||
150 | $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); | ||
151 | } | ||
152 | } | ||
131 | } | 153 | } |
132 | 154 | ||
133 | /** | 155 | /** |
diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index 2ca1683b..8bf7d92e 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php | |||
@@ -5,6 +5,7 @@ namespace Wallabag\ImportBundle\Import; | |||
5 | use Wallabag\CoreBundle\Entity\Entry; | 5 | use Wallabag\CoreBundle\Entity\Entry; |
6 | use Wallabag\UserBundle\Entity\User; | 6 | use Wallabag\UserBundle\Entity\User; |
7 | use Wallabag\CoreBundle\Helper\ContentProxy; | 7 | use Wallabag\CoreBundle\Helper\ContentProxy; |
8 | use Wallabag\CoreBundle\Event\EntrySavedEvent; | ||
8 | 9 | ||
9 | abstract class BrowserImport extends AbstractImport | 10 | abstract class BrowserImport extends AbstractImport |
10 | { | 11 | { |
@@ -81,6 +82,7 @@ abstract class BrowserImport extends AbstractImport | |||
81 | protected function parseEntries($entries) | 82 | protected function parseEntries($entries) |
82 | { | 83 | { |
83 | $i = 1; | 84 | $i = 1; |
85 | $entryToBeFlushed = []; | ||
84 | 86 | ||
85 | foreach ($entries as $importedEntry) { | 87 | foreach ($entries as $importedEntry) { |
86 | if ((array) $importedEntry !== $importedEntry) { | 88 | if ((array) $importedEntry !== $importedEntry) { |
@@ -93,14 +95,29 @@ abstract class BrowserImport extends AbstractImport | |||
93 | continue; | 95 | continue; |
94 | } | 96 | } |
95 | 97 | ||
98 | // @see AbstractImport | ||
99 | $entryToBeFlushed[] = $entry; | ||
100 | |||
96 | // flush every 20 entries | 101 | // flush every 20 entries |
97 | if (($i % 20) === 0) { | 102 | if (($i % 20) === 0) { |
98 | $this->em->flush(); | 103 | $this->em->flush(); |
104 | |||
105 | foreach ($entryToBeFlushed as $entry) { | ||
106 | $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); | ||
107 | } | ||
108 | |||
109 | $entryToBeFlushed = []; | ||
99 | } | 110 | } |
100 | ++$i; | 111 | ++$i; |
101 | } | 112 | } |
102 | 113 | ||
103 | $this->em->flush(); | 114 | $this->em->flush(); |
115 | |||
116 | if (!empty($entryToBeFlushed)) { | ||
117 | foreach ($entryToBeFlushed as $entry) { | ||
118 | $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry)); | ||
119 | } | ||
120 | } | ||
104 | } | 121 | } |
105 | 122 | ||
106 | /** | 123 | /** |
diff --git a/src/Wallabag/ImportBundle/Import/PinboardImport.php b/src/Wallabag/ImportBundle/Import/PinboardImport.php new file mode 100644 index 00000000..9bcfbc36 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/PinboardImport.php | |||
@@ -0,0 +1,143 @@ | |||
1 | <?php | ||
2 | |||
3 | namespace Wallabag\ImportBundle\Import; | ||
4 | |||
5 | use Wallabag\CoreBundle\Entity\Entry; | ||
6 | |||
7 | class PinboardImport extends AbstractImport | ||
8 | { | ||
9 | private $filepath; | ||
10 | |||
11 | /** | ||
12 | * {@inheritdoc} | ||
13 | */ | ||
14 | public function getName() | ||
15 | { | ||
16 | return 'Pinboard'; | ||
17 | } | ||
18 | |||
19 | /** | ||
20 | * {@inheritdoc} | ||
21 | */ | ||
22 | public function getUrl() | ||
23 | { | ||
24 | return 'import_pinboard'; | ||
25 | } | ||
26 | |||
27 | /** | ||
28 | * {@inheritdoc} | ||
29 | */ | ||
30 | public function getDescription() | ||
31 | { | ||
32 | return 'import.pinboard.description'; | ||
33 | } | ||
34 | |||
35 | /** | ||
36 | * Set file path to the json file. | ||
37 | * | ||
38 | * @param string $filepath | ||
39 | */ | ||
40 | public function setFilepath($filepath) | ||
41 | { | ||
42 | $this->filepath = $filepath; | ||
43 | |||
44 | return $this; | ||
45 | } | ||
46 | |||
47 | /** | ||
48 | * {@inheritdoc} | ||
49 | */ | ||
50 | public function import() | ||
51 | { | ||
52 | if (!$this->user) { | ||
53 | $this->logger->error('PinboardImport: user is not defined'); | ||
54 | |||
55 | return false; | ||
56 | } | ||
57 | |||
58 | if (!file_exists($this->filepath) || !is_readable($this->filepath)) { | ||
59 | $this->logger->error('PinboardImport: unable to read file', ['filepath' => $this->filepath]); | ||
60 | |||
61 | return false; | ||
62 | } | ||
63 | |||
64 | $data = json_decode(file_get_contents($this->filepath), true); | ||
65 | |||
66 | if (empty($data)) { | ||
67 | $this->logger->error('PinboardImport: no entries in imported file'); | ||
68 | |||
69 | return false; | ||
70 | } | ||
71 | |||
72 | if ($this->producer) { | ||
73 | $this->parseEntriesForProducer($data); | ||
74 | |||
75 | return true; | ||
76 | } | ||
77 | |||
78 | $this->parseEntries($data); | ||
79 | |||
80 | return true; | ||
81 | } | ||
82 | |||
83 | /** | ||
84 | * {@inheritdoc} | ||
85 | */ | ||
86 | public function parseEntry(array $importedEntry) | ||
87 | { | ||
88 | $existingEntry = $this->em | ||
89 | ->getRepository('WallabagCoreBundle:Entry') | ||
90 | ->findByUrlAndUserId($importedEntry['href'], $this->user->getId()); | ||
91 | |||
92 | if (false !== $existingEntry) { | ||
93 | ++$this->skippedEntries; | ||
94 | |||
95 | return; | ||
96 | } | ||
97 | |||
98 | $data = [ | ||
99 | 'title' => $importedEntry['description'], | ||
100 | 'url' => $importedEntry['href'], | ||
101 | 'content_type' => '', | ||
102 | 'language' => '', | ||
103 | 'is_archived' => ('no' === $importedEntry['toread']) || $this->markAsRead, | ||
104 | 'is_starred' => false, | ||
105 | 'created_at' => $importedEntry['time'], | ||
106 | 'tags' => explode(' ', $importedEntry['tags']), | ||
107 | ]; | ||
108 | |||
109 | $entry = new Entry($this->user); | ||
110 | $entry->setUrl($data['url']); | ||
111 | $entry->setTitle($data['title']); | ||
112 | |||
113 | // update entry with content (in case fetching failed, the given entry will be return) | ||
114 | $entry = $this->fetchContent($entry, $data['url'], $data); | ||
115 | |||
116 | if (!empty($data['tags'])) { | ||
117 | $this->contentProxy->assignTagsToEntry( | ||
118 | $entry, | ||
119 | $data['tags'], | ||
120 | $this->em->getUnitOfWork()->getScheduledEntityInsertions() | ||
121 | ); | ||
122 | } | ||
123 | |||
124 | $entry->setArchived($data['is_archived']); | ||
125 | $entry->setStarred($data['is_starred']); | ||
126 | $entry->setCreatedAt(new \DateTime($data['created_at'])); | ||
127 | |||
128 | $this->em->persist($entry); | ||
129 | ++$this->importedEntries; | ||
130 | |||
131 | return $entry; | ||
132 | } | ||
133 | |||
134 | /** | ||
135 | * {@inheritdoc} | ||
136 | */ | ||
137 | protected function setEntryAsRead(array $importedEntry) | ||
138 | { | ||
139 | $importedEntry['toread'] = 'no'; | ||
140 | |||
141 | return $importedEntry; | ||
142 | } | ||
143 | } | ||
diff --git a/src/Wallabag/ImportBundle/Import/PocketImport.php b/src/Wallabag/ImportBundle/Import/PocketImport.php index 327e2500..33093480 100644 --- a/src/Wallabag/ImportBundle/Import/PocketImport.php +++ b/src/Wallabag/ImportBundle/Import/PocketImport.php | |||
@@ -2,8 +2,6 @@ | |||
2 | 2 | ||
3 | namespace Wallabag\ImportBundle\Import; | 3 | namespace Wallabag\ImportBundle\Import; |
4 | 4 | ||
5 | use Psr\Log\NullLogger; | ||
6 | use Doctrine\ORM\EntityManager; | ||
7 | use GuzzleHttp\Client; | 5 | use GuzzleHttp\Client; |
8 | use GuzzleHttp\Exception\RequestException; | 6 | use GuzzleHttp\Exception\RequestException; |
9 | use Wallabag\CoreBundle\Entity\Entry; | 7 | use Wallabag\CoreBundle\Entity\Entry; |
@@ -16,13 +14,6 @@ class PocketImport extends AbstractImport | |||
16 | 14 | ||
17 | const NB_ELEMENTS = 5000; | 15 | const NB_ELEMENTS = 5000; |
18 | 16 | ||
19 | public function __construct(EntityManager $em, ContentProxy $contentProxy) | ||
20 | { | ||
21 | $this->em = $em; | ||
22 | $this->contentProxy = $contentProxy; | ||
23 | $this->logger = new NullLogger(); | ||
24 | } | ||
25 | |||
26 | /** | 17 | /** |
27 | * Only used for test purpose. | 18 | * Only used for test purpose. |
28 | * | 19 | * |