namespace Wallabag\ImportBundle\Import;
-use Psr\Log\LoggerInterface;
-use Psr\Log\NullLogger;
-use Doctrine\ORM\EntityManager;
use Wallabag\CoreBundle\Entity\Entry;
-use Wallabag\UserBundle\Entity\User;
-use Wallabag\CoreBundle\Helper\ContentProxy;
+use Wallabag\CoreBundle\Event\EntrySavedEvent;
-class BrowserImport implements ImportInterface
+abstract class BrowserImport extends AbstractImport
{
- protected $user;
- protected $em;
- protected $logger;
- protected $contentProxy;
- protected $skippedEntries = 0;
- protected $importedEntries = 0;
- protected $totalEntries = 0;
protected $filepath;
- protected $markAsRead;
- private $nbEntries;
-
- public function __construct(EntityManager $em, ContentProxy $contentProxy)
- {
- $this->em = $em;
- $this->logger = new NullLogger();
- $this->contentProxy = $contentProxy;
- }
-
- public function setLogger(LoggerInterface $logger)
- {
- $this->logger = $logger;
- }
-
- /**
- * We define the user in a custom call because on the import command there is no logged in user.
- * So we can't retrieve user from the `security.token_storage` service.
- *
- * @param User $user
- *
- * @return $this
- */
- public function setUser(User $user)
- {
- $this->user = $user;
-
- return $this;
- }
/**
* {@inheritdoc}
*/
- public function getName()
- {
- return 'Firefox & Google Chrome';
- }
+ abstract public function getName();
/**
* {@inheritdoc}
*/
- public function getUrl()
- {
- return 'import_browser';
- }
+ abstract public function getUrl();
/**
* {@inheritdoc}
*/
- public function getDescription()
- {
- return 'import.browser.description';
- }
+ abstract public function getDescription();
/**
* {@inheritdoc}
public function import()
{
if (!$this->user) {
- $this->logger->error('WallabagImport: user is not defined');
+ $this->logger->error('Wallabag Browser Import: user is not defined');
return false;
}
if (!file_exists($this->filepath) || !is_readable($this->filepath)) {
- $this->logger->error('WallabagImport: unable to read file', ['filepath' => $this->filepath]);
+ $this->logger->error('Wallabag Browser Import: unable to read file', ['filepath' => $this->filepath]);
return false;
}
$data = json_decode(file_get_contents($this->filepath), true);
if (empty($data)) {
+ $this->logger->error('Wallabag Browser: no entries in imported file');
+
return false;
}
- $this->nbEntries = 1;
+ if ($this->producer) {
+ $this->parseEntriesForProducer($data);
+
+ return true;
+ }
+
$this->parseEntries($data);
- $this->em->flush();
return true;
}
- private function parseEntries($data)
+ /**
+ * Set file path to the json file.
+ *
+ * @param string $filepath
+ */
+ public function setFilepath($filepath)
{
- foreach ($data as $importedEntry) {
- $this->parseEntry($importedEntry);
- }
- $this->totalEntries += count($data);
+ $this->filepath = $filepath;
+
+ return $this;
}
- private function parseEntry($importedEntry)
+ /**
+ * {@inheritdoc}
+ */
+ public function parseEntry(array $importedEntry)
{
- if (!is_array($importedEntry)) {
- return;
- }
+ if ((!array_key_exists('guid', $importedEntry) || (!array_key_exists('id', $importedEntry))) && \is_array(reset($importedEntry))) {
+ if ($this->producer) {
+ $this->parseEntriesForProducer($importedEntry);
- /* Firefox uses guid while Chrome uses id */
+ return;
+ }
- if ((!key_exists('guid', $importedEntry) || (!key_exists('id', $importedEntry))) && is_array(reset($importedEntry))) {
$this->parseEntries($importedEntry);
return;
}
- if (key_exists('children', $importedEntry)) {
+
+ if (array_key_exists('children', $importedEntry)) {
+ if ($this->producer) {
+ $this->parseEntriesForProducer($importedEntry['children']);
+
+ return;
+ }
+
$this->parseEntries($importedEntry['children']);
return;
}
- if (key_exists('uri', $importedEntry) || key_exists('url', $importedEntry)) {
-
- /* Firefox uses uri while Chrome uses url */
- $firefox = key_exists('uri', $importedEntry);
+ if (!array_key_exists('uri', $importedEntry) && !array_key_exists('url', $importedEntry)) {
+ return;
+ }
- $existingEntry = $this->em
- ->getRepository('WallabagCoreBundle:Entry')
- ->findByUrlAndUserId(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], $this->user->getId());
+ $url = array_key_exists('uri', $importedEntry) ? $importedEntry['uri'] : $importedEntry['url'];
- if (false !== $existingEntry) {
- ++$this->skippedEntries;
+ $existingEntry = $this->em
+ ->getRepository('WallabagCoreBundle:Entry')
+ ->findByUrlAndUserId($url, $this->user->getId());
- return;
- }
+ if (false !== $existingEntry) {
+ ++$this->skippedEntries;
- if (false === parse_url(($firefox) ? $importedEntry['uri'] : $importedEntry['url']) || false === filter_var(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], FILTER_VALIDATE_URL)) {
- $this->logger->warning('Imported URL '.($firefox) ? $importedEntry['uri'] : $importedEntry['url'].' is not valid');
- ++$this->skippedEntries;
+ return;
+ }
- return;
- }
+ $data = $this->prepareEntry($importedEntry);
- try {
- $entry = $this->contentProxy->updateEntry(
- new Entry($this->user),
- ($firefox) ? $importedEntry['uri'] : $importedEntry['url']
- );
- } catch (\Exception $e) {
- $this->logger->warning('Error while saving '.($firefox) ? $importedEntry['uri'] : $importedEntry['url']);
- ++$this->skippedEntries;
+ $entry = new Entry($this->user);
+ $entry->setUrl($data['url']);
+ $entry->setTitle($data['title']);
- return;
- }
+ // update entry with content (in case fetching failed, the given entry will be return)
+ $this->fetchContent($entry, $data['url'], $data);
- $entry->setArchived($this->markAsRead);
+ if (array_key_exists('tags', $data)) {
+ $this->tagsAssigner->assignTagsToEntry(
+ $entry,
+ $data['tags']
+ );
+ }
- $this->em->persist($entry);
- ++$this->importedEntries;
+ $entry->setArchived($data['is_archived']);
- // flush every 20 entries
- if (($this->nbEntries % 20) === 0) {
- $this->em->flush();
- $this->em->clear($entry);
- }
- ++$this->nbEntries;
+ if (!empty($data['created_at'])) {
+ $dt = new \DateTime();
+ $entry->setCreatedAt($dt->setTimestamp($data['created_at']));
}
+
+ $this->em->persist($entry);
+ ++$this->importedEntries;
+
+ return $entry;
}
/**
- * Set whether articles must be all marked as read.
- *
- * @param bool $markAsRead
+ * Parse and insert all given entries.
*
- * @return $this
+ * @param array $entries
*/
- public function setMarkAsRead($markAsRead)
+ protected function parseEntries(array $entries)
{
- $this->markAsRead = $markAsRead;
+ $i = 1;
+ $entryToBeFlushed = [];
- return $this;
+ foreach ($entries as $importedEntry) {
+ if ((array) $importedEntry !== $importedEntry) {
+ continue;
+ }
+
+ $entry = $this->parseEntry($importedEntry);
+
+ if (null === $entry) {
+ continue;
+ }
+
+ // @see AbstractImport
+ $entryToBeFlushed[] = $entry;
+
+ // flush every 20 entries
+ if (0 === ($i % 20)) {
+ $this->em->flush();
+
+ foreach ($entryToBeFlushed as $entry) {
+ $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry));
+ }
+
+ $entryToBeFlushed = [];
+ }
+ ++$i;
+ }
+
+ $this->em->flush();
+
+ if (!empty($entryToBeFlushed)) {
+ foreach ($entryToBeFlushed as $entry) {
+ $this->eventDispatcher->dispatch(EntrySavedEvent::NAME, new EntrySavedEvent($entry));
+ }
+ }
}
/**
- * Set file path to the json file.
+ * Parse entries and send them to the queue.
+ * It should just be a simple loop on all item, no call to the database should be done
+ * to speedup queuing.
*
- * @param string $filepath
+ * Faster parse entries for Producer.
+ * We don't care to make check at this time. They'll be done by the consumer.
*
- * @return $this
+ * @param array $entries
*/
- public function setFilepath($filepath)
+ protected function parseEntriesForProducer(array $entries)
{
- $this->filepath = $filepath;
+ foreach ($entries as $importedEntry) {
+ if ((array) $importedEntry !== $importedEntry) {
+ continue;
+ }
- return $this;
+ // set userId for the producer (it won't know which user is connected)
+ $importedEntry['userId'] = $this->user->getId();
+
+ if ($this->markAsRead) {
+ $importedEntry = $this->setEntryAsRead($importedEntry);
+ }
+
+ ++$this->queuedEntries;
+
+ $this->producer->publish(json_encode($importedEntry));
+ }
}
/**
* {@inheritdoc}
*/
- public function getSummary()
+ protected function setEntryAsRead(array $importedEntry)
{
- return [
- 'skipped' => $this->skippedEntries,
- 'imported' => $this->importedEntries,
- ];
+ $importedEntry['is_archived'] = 1;
+
+ return $importedEntry;
}
+
+ abstract protected function prepareEntry(array $entry = []);
}