From 9ab024b4f5388e2a41c50a8a2b79e4033788782a Mon Sep 17 00:00:00 2001 From: Jeremy Benoist Date: Fri, 4 Nov 2016 22:44:31 +0100 Subject: [PATCH] Add Pinboard import --- app/config/config.yml | 13 ++ .../Resources/translations/messages.da.yml | 4 + .../Resources/translations/messages.de.yml | 4 + .../Resources/translations/messages.en.yml | 4 + .../Resources/translations/messages.es.yml | 4 + .../Resources/translations/messages.fa.yml | 4 + .../Resources/translations/messages.fr.yml | 4 + .../Resources/translations/messages.it.yml | 4 + .../Resources/translations/messages.oc.yml | 4 + .../Resources/translations/messages.pl.yml | 4 + .../Resources/translations/messages.pt.yml | 25 +++ .../Resources/translations/messages.ro.yml | 4 + .../Resources/translations/messages.tr.yml | 4 + .../ImportBundle/Command/ImportCommand.php | 5 +- .../Command/RedisWorkerCommand.php | 2 +- .../Controller/ImportController.php | 2 + .../Controller/PinboardController.php | 77 +++++++ .../ImportBundle/Import/PinboardImport.php | 143 +++++++++++++ .../ImportBundle/Resources/config/rabbit.yml | 8 + .../ImportBundle/Resources/config/redis.yml | 21 ++ .../Resources/config/services.yml | 11 + .../Resources/views/Pinboard/index.html.twig | 45 ++++ .../Controller/PinboardControllerTest.php | 197 ++++++++++++++++++ .../ImportBundle/fixtures/pinboard_export | 5 + 24 files changed, 596 insertions(+), 2 deletions(-) create mode 100644 src/Wallabag/ImportBundle/Controller/PinboardController.php create mode 100644 src/Wallabag/ImportBundle/Import/PinboardImport.php create mode 100644 src/Wallabag/ImportBundle/Resources/views/Pinboard/index.html.twig create mode 100644 tests/Wallabag/ImportBundle/Controller/PinboardControllerTest.php create mode 100644 tests/Wallabag/ImportBundle/fixtures/pinboard_export diff --git a/app/config/config.yml b/app/config/config.yml index 7f24244d..ff0f5a0b 100644 --- a/app/config/config.yml +++ b/app/config/config.yml @@ -266,6 +266,11 @@ old_sound_rabbit_mq: exchange_options: name: 'wallabag.import.readability' type: topic + import_pinboard: + connection: default + exchange_options: + name: 'wallabag.import.pinboard' + type: topic import_instapaper: connection: default exchange_options: @@ -316,6 +321,14 @@ old_sound_rabbit_mq: queue_options: name: 'wallabag.import.instapaper' callback: wallabag_import.consumer.amqp.instapaper + import_pinboard: + connection: default + exchange_options: + name: 'wallabag.import.pinboard' + type: topic + queue_options: + name: 'wallabag.import.pinboard' + callback: wallabag_import.consumer.amqp.pinboard import_wallabag_v1: connection: default exchange_options: diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml index aeae6bcf..14b528f9 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml @@ -381,6 +381,10 @@ import: # page_title: 'Import > Instapaper' # description: 'This importer will import all your Instapaper articles. On the settings (https://www.instapaper.com/user) page, click on "Download .CSV file" in the "Export" section. A CSV file will be downloaded (like "instapaper-export.csv").' # how_to: 'Please select your Instapaper export and click on the below button to upload and import it.' + pinboard: + # page_title: "Import > Pinboard" + # description: 'This importer will import all your Instapaper articles. On the backup (https://pinboard.in/settings/backup) page, click on "JSON" in the "Bookmarks" section. A JSON file will be downloaded (like "pinboard_export").' + # how_to: 'Please select your Pinboard export and click on the below button to upload and import it.' developer: # page_title: 'Developer' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml index 2105d02d..926a1846 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml @@ -381,6 +381,10 @@ import: page_title: 'Aus Instapaper importieren' description: 'Dieser Import wird all deine Instapaper Artikel importieren. Auf der Einstellungsseite (https://www.instapaper.com/user) klickst du auf "Download .CSV Datei" in dem Abschnitt "Export". Eine CSV Datei wird heruntergeladen (z.B. "instapaper-export.csv").' how_to: "Bitte wähle deine Instapaper Sicherungsdatei aus und klicke den nachfolgenden Button zum Importieren." + pinboard: + # page_title: "Import > Pinboard" + # description: 'This importer will import all your Instapaper articles. On the backup (https://pinboard.in/settings/backup) page, click on "JSON" in the "Bookmarks" section. A JSON file will be downloaded (like "pinboard_export").' + # how_to: 'Please select your Pinboard export and click on the below button to upload and import it.' developer: page_title: 'Entwickler' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml index 2bb95728..e950857d 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml @@ -381,6 +381,10 @@ import: page_title: 'Import > Instapaper' description: 'This importer will import all your Instapaper articles. On the settings (https://www.instapaper.com/user) page, click on "Download .CSV file" in the "Export" section. A CSV file will be downloaded (like "instapaper-export.csv").' how_to: 'Please select your Instapaper export and click on the below button to upload and import it.' + pinboard: + page_title: "Import > Pinboard" + description: 'This importer will import all your Instapaper articles. On the backup (https://pinboard.in/settings/backup) page, click on "JSON" in the "Bookmarks" section. A JSON file will be downloaded (like "pinboard_export").' + how_to: 'Please select your Pinboard export and click on the below button to upload and import it.' developer: page_title: 'Developer' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml index ca3db487..8b555cc2 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml @@ -381,6 +381,10 @@ import: page_title: 'Importar > Instapaper' # description: 'This importer will import all your Instapaper articles. On the settings (https://www.instapaper.com/user) page, click on "Download .CSV file" in the "Export" section. A CSV file will be downloaded (like "instapaper-export.csv").' # how_to: 'Please select your Instapaper export and click on the below button to upload and import it.' + pinboard: + page_title: "Importar > Pinboard" + # description: 'This importer will import all your Instapaper articles. On the backup (https://pinboard.in/settings/backup) page, click on "JSON" in the "Bookmarks" section. A JSON file will be downloaded (like "pinboard_export").' + # how_to: 'Please select your Pinboard export and click on the below button to upload and import it.' developer: page_title: 'Promotor' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml index 1914215a..3d7e8078 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml @@ -381,6 +381,10 @@ import: page_title: 'درون‌ریزی > Instapaper' # description: 'This importer will import all your Instapaper articles. On the settings (https://www.instapaper.com/user) page, click on "Download .CSV file" in the "Export" section. A CSV file will be downloaded (like "instapaper-export.csv").' # how_to: 'Please select your Instapaper export and click on the below button to upload and import it.' + pinboard: + # page_title: "Import > Pinboard" + # description: 'This importer will import all your Instapaper articles. On the backup (https://pinboard.in/settings/backup) page, click on "JSON" in the "Bookmarks" section. A JSON file will be downloaded (like "pinboard_export").' + # how_to: 'Please select your Pinboard export and click on the below button to upload and import it.' developer: # page_title: 'Developer' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml index 60fa9a39..833f97bb 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml @@ -381,6 +381,10 @@ import: page_title: "Import > Instapaper" description: "Sur la page des paramètres (https://www.instapaper.com/user), cliquez sur « Download .CSV file » dans la section « Export ». Un fichier CSV sera téléchargé (« instapaper-export.csv »)." how_to: "Choisissez le fichier de votre export Instapaper et cliquez sur le bouton ci-dessous pour l’importer." + pinboard: + page_title: "Import > Pinboard" + description: "Sur la page « Backup » (https://pinboard.in/settings/backup), cliquez sur « JSON » dans la section « Bookmarks ». Un fichier json (sans extension) sera téléchargé (« pinboard_export »)." + how_to: "Choisissez le fichier de votre export Pinboard et cliquez sur le bouton ci-dessous pour l’importer." developer: page_title: "Développeur" diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml index 7f401684..197d5e07 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml @@ -381,6 +381,10 @@ import: page_title: 'Importa da > Instapaper' # description: 'This importer will import all your Instapaper articles. On the settings (https://www.instapaper.com/user) page, click on "Download .CSV file" in the "Export" section. A CSV file will be downloaded (like "instapaper-export.csv").' # how_to: 'Please select your Instapaper export and click on the below button to upload and import it.' + pinboard: + page_title: "Importa da > Pinboard" + # description: 'This importer will import all your Instapaper articles. On the backup (https://pinboard.in/settings/backup) page, click on "JSON" in the "Bookmarks" section. A JSON file will be downloaded (like "pinboard_export").' + # how_to: 'Please select your Pinboard export and click on the below button to upload and import it.' developer: page_title: 'Sviluppatori' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml index 4485c3bc..75268e00 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml @@ -381,6 +381,10 @@ import: page_title: 'Importar > Instapaper' description: "Aquesta aisina importarà totas vòstres articles d'Instapaper. Sus la pagina de paramètres (https://www.instapaper.com/user), clicatz sus \"Download .CSV file\" dins la seccion \"Export\". Un fichièr CSV serà telecargat (aital \"instapaper-export.csv\")." how_to: "Mercés de causir vòstre fichièr Instapaper e de clicar sul boton dejós per lo telecargar e l'importar" + pinboard: + # page_title: "Import > Pinboard" + # description: 'This importer will import all your Instapaper articles. On the backup (https://pinboard.in/settings/backup) page, click on "JSON" in the "Bookmarks" section. A JSON file will be downloaded (like "pinboard_export").' + # how_to: 'Please select your Pinboard export and click on the below button to upload and import it.' developer: page_title: 'Desvolopaire' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml index ca9015bf..40ded0be 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml @@ -381,6 +381,10 @@ import: page_title: 'Import > Instapaper' description: 'Ten importer, zaimportuje wszystkie twoje artykuły z Instapaper. W ustawieniach (https://www.instapaper.com/user), kliknij na "Download .CSV file" w sekcji "Export". Otrzymasz plik CSV.' how_to: 'Wybierz swój plik eksportu z Instapaper i kliknij poniższy przycisk, aby go załadować.' + pinboard: + # page_title: "Import > Pinboard" + # description: 'This importer will import all your Instapaper articles. On the backup (https://pinboard.in/settings/backup) page, click on "JSON" in the "Bookmarks" section. A JSON file will be downloaded (like "pinboard_export").' + # how_to: 'Please select your Pinboard export and click on the below button to upload and import it.' developer: page_title: 'Deweloper' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.pt.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.pt.yml index c1c60430..ed371cbc 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.pt.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.pt.yml @@ -71,6 +71,7 @@ config: 300_word: 'Posso ler ~300 palavras por minuto' 400_word: 'Posso ler ~400 palavras por minuto' pocket_consumer_key_label: 'Chave do consumidor do Pocket para importar conteúdo' + # android_configuration: Configure your Android application form_rss: description: 'Feeds RSS providos pelo wallabag permitem que você leia seus artigos salvos em seu leitor de RSS favorito. Você precisa gerar um token primeiro.' token_label: 'Token RSS' @@ -88,6 +89,18 @@ config: name_label: 'Nome' email_label: 'E-mail' twoFactorAuthentication_label: 'Autenticação de dois passos' + delete: + # title: Delete my account (a.k.a danger zone) + # description: If you remove your account, ALL your articles, ALL your tags, ALL your annotations and your account will be PERMANENTLY removed (it can't be UNDONE). You'll then be logged out. + # confirm: Are you really sure? (THIS CAN'T BE UNDONE) + # button: Delete my account + reset: + # title: Reset area (a.k.a danger zone) + # description: By hiting buttons below you'll have ability to remove some informations from your account. Be aware that these actions are IRREVERSIBLE. + # annotations: Remove ALL annotations + # tags: Remove ALL tags + # entries: Remove ALL entries + # confirm: Are you really really sure? (THIS CAN'T BE UNDONE) form_password: old_password_label: 'Senha atual' new_password_label: 'Nova senha' @@ -355,6 +368,7 @@ import: how_to: 'Por favor, selecione sua exportação do Readability e clique no botão abaixo para importá-la.' worker: enabled: "A importação é feita assíncronamente. Uma vez que a tarefa de importação é iniciada, um trabalho externo pode executar tarefas uma por vez. O serviço atual é:" + # download_images_warning: "You enabled downloading images for your articles. Combined with classic import it can take ages to proceed (or maybe failed). We strongly recommend to enable asynchronous import to avoid errors." firefox: page_title: 'Importar > Firefox' description: "Com este importador você importa todos os favoritos de seu Firefox. Somente vá até seus favoritos (Ctrl+Maj+O), e em \"Importar e Backup\" e escolha \"Backup...\". Você terá então um arquivo .json." @@ -367,6 +381,10 @@ import: page_title: 'Importar > Instapaper' description: 'Este importador pode importar todos os artigos do seu Instapaper. Nas página de configurações (https://www.instapaper.com/user), clique em "Download .CSV file" na seção "Export". Um arquivo CSV será baixado (algo como "instapaper-export.csv").' how_to: 'Por favor, selecione sua exportação do seu Instapaper e clique no botão abaixo para importá-la.' + pinboard: + # page_title: "Import > Pinboard" + # description: 'This importer will import all your Instapaper articles. On the backup (https://pinboard.in/settings/backup) page, click on "JSON" in the "Bookmarks" section. A JSON file will be downloaded (like "pinboard_export").' + # how_to: 'Please select your Pinboard export and click on the below button to upload and import it.' developer: page_title: 'Desenvolvedor' @@ -445,16 +463,23 @@ user: delete_confirm: 'Tem certeza?' back_to_list: 'Voltar para a lista' +error: + # page_title: An error occurred + flashes: config: notice: config_saved: 'Configiração salva. Alguns parâmetros podem ser considerados depois da desconexão.' password_updated: 'Senha atualizada' password_not_updated_demo: 'Em modo de demonstração, você não pode alterar a senha deste usuário.' + # user_updated: 'Information updated' rss_updated: 'Informação de RSS atualizada' tagging_rules_updated: 'Regras de tags atualizadas' tagging_rules_deleted: 'Regra de tag apagada' rss_token_updated: 'Token RSS atualizado' + # annotations_reset: Annotations reset + # tags_reset: Tags reset + # entries_reset: Entries reset entry: notice: entry_already_saved: 'Entrada já foi salva em %date%' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml index 50f1b6a2..a3c0e266 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml @@ -381,6 +381,10 @@ import: # page_title: 'Import > Instapaper' # description: 'This importer will import all your Instapaper articles. On the settings (https://www.instapaper.com/user) page, click on "Download .CSV file" in the "Export" section. A CSV file will be downloaded (like "instapaper-export.csv").' # how_to: 'Please select your Instapaper export and click on the below button to upload and import it.' + pinboard: + # page_title: "Import > Pinboard" + # description: 'This importer will import all your Instapaper articles. On the backup (https://pinboard.in/settings/backup) page, click on "JSON" in the "Bookmarks" section. A JSON file will be downloaded (like "pinboard_export").' + # how_to: 'Please select your Pinboard export and click on the below button to upload and import it.' developer: # page_title: 'Developer' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml index 07939ebc..a6d2e137 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml @@ -381,6 +381,10 @@ import: page_title: 'İçe Aktar > Instapaper' # description: 'This importer will import all your Instapaper articles. On the settings (https://www.instapaper.com/user) page, click on "Download .CSV file" in the "Export" section. A CSV file will be downloaded (like "instapaper-export.csv").' # how_to: 'Please select your Instapaper export and click on the below button to upload and import it.' + pinboard: + # page_title: "Import > Pinboard" + # description: 'This importer will import all your Instapaper articles. On the backup (https://pinboard.in/settings/backup) page, click on "JSON" in the "Bookmarks" section. A JSON file will be downloaded (like "pinboard_export").' + # how_to: 'Please select your Pinboard export and click on the below button to upload and import it.' developer: # page_title: 'Developer' diff --git a/src/Wallabag/ImportBundle/Command/ImportCommand.php b/src/Wallabag/ImportBundle/Command/ImportCommand.php index e423ffae..28d01715 100644 --- a/src/Wallabag/ImportBundle/Command/ImportCommand.php +++ b/src/Wallabag/ImportBundle/Command/ImportCommand.php @@ -17,7 +17,7 @@ class ImportCommand extends ContainerAwareCommand ->setDescription('Import entries from a JSON export') ->addArgument('userId', InputArgument::REQUIRED, 'User ID to populate') ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file') - ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: v1, v2, instapaper, readability, firefox or chrome', 'v1') + ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: v1, v2, instapaper, pinboard, readability, firefox or chrome', 'v1') ->addOption('markAsRead', null, InputArgument::OPTIONAL, 'Mark all entries as read', false) ; } @@ -56,6 +56,9 @@ class ImportCommand extends ContainerAwareCommand case 'instapaper': $import = $this->getContainer()->get('wallabag_import.instapaper.import'); break; + case 'pinboard': + $import = $this->getContainer()->get('wallabag_import.pinboard.import'); + break; default: $import = $this->getContainer()->get('wallabag_import.wallabag_v1.import'); } diff --git a/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php b/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php index c2c11f11..f793a314 100644 --- a/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php +++ b/src/Wallabag/ImportBundle/Command/RedisWorkerCommand.php @@ -17,7 +17,7 @@ class RedisWorkerCommand extends ContainerAwareCommand $this ->setName('wallabag:import:redis-worker') ->setDescription('Launch Redis worker') - ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket, readability, firefox, chrome or instapaper') + ->addArgument('serviceName', InputArgument::REQUIRED, 'Service to use: wallabag_v1, wallabag_v2, pocket, readability, pinboard, firefox, chrome or instapaper') ->addOption('maxIterations', '', InputOption::VALUE_OPTIONAL, 'Number of iterations before stoping', false) ; } diff --git a/src/Wallabag/ImportBundle/Controller/ImportController.php b/src/Wallabag/ImportBundle/Controller/ImportController.php index 15de75ff..237c748e 100644 --- a/src/Wallabag/ImportBundle/Controller/ImportController.php +++ b/src/Wallabag/ImportBundle/Controller/ImportController.php @@ -42,6 +42,7 @@ class ImportController extends Controller + $this->getTotalMessageInRabbitQueue('firefox') + $this->getTotalMessageInRabbitQueue('chrome') + $this->getTotalMessageInRabbitQueue('instapaper') + + $this->getTotalMessageInRabbitQueue('pinboard') ; } catch (\Exception $e) { $rabbitNotInstalled = true; @@ -57,6 +58,7 @@ class ImportController extends Controller + $redis->llen('wallabag.import.firefox') + $redis->llen('wallabag.import.chrome') + $redis->llen('wallabag.import.instapaper') + + $redis->llen('wallabag.import.pinboard') ; } catch (\Exception $e) { $redisNotInstalled = true; diff --git a/src/Wallabag/ImportBundle/Controller/PinboardController.php b/src/Wallabag/ImportBundle/Controller/PinboardController.php new file mode 100644 index 00000000..9c3f98d6 --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/PinboardController.php @@ -0,0 +1,77 @@ +createForm(UploadImportType::class); + $form->handleRequest($request); + + $pinboard = $this->get('wallabag_import.pinboard.import'); + $pinboard->setUser($this->getUser()); + + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $pinboard->setProducer($this->get('old_sound_rabbit_mq.import_pinboard_producer')); + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $pinboard->setProducer($this->get('wallabag_import.producer.redis.pinboard')); + } + + if ($form->isValid()) { + $file = $form->get('file')->getData(); + $markAsRead = $form->get('mark_as_read')->getData(); + $name = 'pinboard_'.$this->getUser()->getId().'.json'; + + if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { + $res = $pinboard + ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) + ->setMarkAsRead($markAsRead) + ->import(); + + $message = 'flashes.import.notice.failed'; + + if (true === $res) { + $summary = $pinboard->getSummary(); + $message = $this->get('translator')->trans('flashes.import.notice.summary', [ + '%imported%' => $summary['imported'], + '%skipped%' => $summary['skipped'], + ]); + + if (0 < $summary['queued']) { + $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [ + '%queued%' => $summary['queued'], + ]); + } + + unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name); + } + + $this->get('session')->getFlashBag()->add( + 'notice', + $message + ); + + return $this->redirect($this->generateUrl('homepage')); + } else { + $this->get('session')->getFlashBag()->add( + 'notice', + 'flashes.import.notice.failed_on_file' + ); + } + } + + return $this->render('WallabagImportBundle:Pinboard:index.html.twig', [ + 'form' => $form->createView(), + 'import' => $pinboard, + ]); + } +} diff --git a/src/Wallabag/ImportBundle/Import/PinboardImport.php b/src/Wallabag/ImportBundle/Import/PinboardImport.php new file mode 100644 index 00000000..9bcfbc36 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/PinboardImport.php @@ -0,0 +1,143 @@ +filepath = $filepath; + + return $this; + } + + /** + * {@inheritdoc} + */ + public function import() + { + if (!$this->user) { + $this->logger->error('PinboardImport: user is not defined'); + + return false; + } + + if (!file_exists($this->filepath) || !is_readable($this->filepath)) { + $this->logger->error('PinboardImport: unable to read file', ['filepath' => $this->filepath]); + + return false; + } + + $data = json_decode(file_get_contents($this->filepath), true); + + if (empty($data)) { + $this->logger->error('PinboardImport: no entries in imported file'); + + return false; + } + + if ($this->producer) { + $this->parseEntriesForProducer($data); + + return true; + } + + $this->parseEntries($data); + + return true; + } + + /** + * {@inheritdoc} + */ + public function parseEntry(array $importedEntry) + { + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId($importedEntry['href'], $this->user->getId()); + + if (false !== $existingEntry) { + ++$this->skippedEntries; + + return; + } + + $data = [ + 'title' => $importedEntry['description'], + 'url' => $importedEntry['href'], + 'content_type' => '', + 'language' => '', + 'is_archived' => ('no' === $importedEntry['toread']) || $this->markAsRead, + 'is_starred' => false, + 'created_at' => $importedEntry['time'], + 'tags' => explode(' ', $importedEntry['tags']), + ]; + + $entry = new Entry($this->user); + $entry->setUrl($data['url']); + $entry->setTitle($data['title']); + + // update entry with content (in case fetching failed, the given entry will be return) + $entry = $this->fetchContent($entry, $data['url'], $data); + + if (!empty($data['tags'])) { + $this->contentProxy->assignTagsToEntry( + $entry, + $data['tags'], + $this->em->getUnitOfWork()->getScheduledEntityInsertions() + ); + } + + $entry->setArchived($data['is_archived']); + $entry->setStarred($data['is_starred']); + $entry->setCreatedAt(new \DateTime($data['created_at'])); + + $this->em->persist($entry); + ++$this->importedEntries; + + return $entry; + } + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['toread'] = 'no'; + + return $importedEntry; + } +} diff --git a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml index a5af5282..e9ecb846 100644 --- a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml +++ b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml @@ -24,6 +24,14 @@ services: - "@wallabag_import.instapaper.import" - "@event_dispatcher" - "@logger" + wallabag_import.consumer.amqp.pinboard: + class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.pinboard.import" + - "@event_dispatcher" + - "@logger" wallabag_import.consumer.amqp.wallabag_v1: class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer arguments: diff --git a/src/Wallabag/ImportBundle/Resources/config/redis.yml b/src/Wallabag/ImportBundle/Resources/config/redis.yml index 5ced4c83..091cdba0 100644 --- a/src/Wallabag/ImportBundle/Resources/config/redis.yml +++ b/src/Wallabag/ImportBundle/Resources/config/redis.yml @@ -42,6 +42,27 @@ services: - "@event_dispatcher" - "@logger" + # pinboard + wallabag_import.queue.redis.pinboard: + class: Simpleue\Queue\RedisQueue + arguments: + - "@wallabag_core.redis.client" + - "wallabag.import.pinboard" + + wallabag_import.producer.redis.pinboard: + class: Wallabag\ImportBundle\Redis\Producer + arguments: + - "@wallabag_import.queue.redis.pinboard" + + wallabag_import.consumer.redis.pinboard: + class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.pinboard.import" + - "@event_dispatcher" + - "@logger" + # pocket wallabag_import.queue.redis.pocket: class: Simpleue\Queue\RedisQueue diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index 64822963..c4fe3f92 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml @@ -71,6 +71,17 @@ services: tags: - { name: wallabag_import.import, alias: instapaper } + wallabag_import.pinboard.import: + class: Wallabag\ImportBundle\Import\PinboardImport + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_core.content_proxy" + - "@event_dispatcher" + calls: + - [ setLogger, [ "@logger" ]] + tags: + - { name: wallabag_import.import, alias: pinboard } + wallabag_import.firefox.import: class: Wallabag\ImportBundle\Import\FirefoxImport arguments: diff --git a/src/Wallabag/ImportBundle/Resources/views/Pinboard/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Pinboard/index.html.twig new file mode 100644 index 00000000..43f196ad --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/views/Pinboard/index.html.twig @@ -0,0 +1,45 @@ +{% extends "WallabagCoreBundle::layout.html.twig" %} + +{% block title %}{{ 'import.pinboard.page_title'|trans }}{% endblock %} + +{% block content %} +
+
+
+ {% include 'WallabagImportBundle:Import:_information.html.twig' %} + +
+
{{ import.description|trans }}
+

{{ 'import.pinboard.how_to'|trans }}

+ +
+ {{ form_start(form, {'method': 'POST'}) }} + {{ form_errors(form) }} +
+
+ {{ form_errors(form.file) }} +
+ {{ form.file.vars.label|trans }} + {{ form_widget(form.file) }} +
+
+ +
+
+
+
{{ 'import.form.mark_as_read_title'|trans }}
+ {{ form_widget(form.mark_as_read) }} + {{ form_label(form.mark_as_read) }} +
+
+ + {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }} + + {{ form_rest(form) }} + +
+
+
+
+
+{% endblock %} diff --git a/tests/Wallabag/ImportBundle/Controller/PinboardControllerTest.php b/tests/Wallabag/ImportBundle/Controller/PinboardControllerTest.php new file mode 100644 index 00000000..84e47b72 --- /dev/null +++ b/tests/Wallabag/ImportBundle/Controller/PinboardControllerTest.php @@ -0,0 +1,197 @@ +logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/pinboard'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + } + + public function testImportPinboardWithRabbitEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1); + + $crawler = $client->request('GET', '/import/pinboard'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + + $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0); + } + + public function testImportPinboardBadFile() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/pinboard'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $data = [ + 'upload_import_file[file]' => '', + ]; + + $client->submit($form, $data); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + } + + public function testImportPinboardWithRedisEnabled() + { + $this->checkRedis(); + $this->logInAs('admin'); + $client = $this->getClient(); + $client->getContainer()->get('craue_config')->set('import_with_redis', 1); + + $crawler = $client->request('GET', '/import/pinboard'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__.'/../fixtures/pinboard_export', 'pinboard.json'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertEquals(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertContains('flashes.import.notice.summary', $body[0]); + + $this->assertNotEmpty($client->getContainer()->get('wallabag_core.redis.client')->lpop('wallabag.import.pinboard')); + + $client->getContainer()->get('craue_config')->set('import_with_redis', 0); + } + + public function testImportPinboardWithFile() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/pinboard'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__.'/../fixtures/pinboard_export', 'pinboard.json'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertEquals(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $content = $client->getContainer() + ->get('doctrine.orm.entity_manager') + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId( + 'https://ma.ttias.be/varnish-explained/', + $this->getLoggedInUserId() + ); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertContains('flashes.import.notice.summary', $body[0]); + + $this->assertNotEmpty($content->getMimetype()); + $this->assertNotEmpty($content->getPreviewPicture()); + $this->assertNotEmpty($content->getLanguage()); + $this->assertEquals(0, count($content->getTags())); + $this->assertInstanceOf(\DateTime::class, $content->getCreatedAt()); + $this->assertEquals('2016-10-26', $content->getCreatedAt()->format('Y-m-d')); + } + + public function testImportPinboardWithFileAndMarkAllAsRead() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/pinboard'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__.'/../fixtures/pinboard_export', 'pinboard-read.json'); + + $data = [ + 'upload_import_file[file]' => $file, + 'upload_import_file[mark_as_read]' => 1, + ]; + + $client->submit($form, $data); + + $this->assertEquals(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $content1 = $client->getContainer() + ->get('doctrine.orm.entity_manager') + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId( + 'https://ilia.ws/files/nginx_torontophpug.pdf', + $this->getLoggedInUserId() + ); + + $this->assertTrue($content1->isArchived()); + + $content2 = $client->getContainer() + ->get('doctrine.orm.entity_manager') + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId( + 'https://developers.google.com/web/updates/2016/07/infinite-scroller', + $this->getLoggedInUserId() + ); + + $this->assertTrue($content2->isArchived()); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertContains('flashes.import.notice.summary', $body[0]); + } + + public function testImportPinboardWithEmptyFile() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/pinboard'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__.'/../fixtures/test.txt', 'test.txt'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertEquals(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertContains('flashes.import.notice.failed', $body[0]); + } +} diff --git a/tests/Wallabag/ImportBundle/fixtures/pinboard_export b/tests/Wallabag/ImportBundle/fixtures/pinboard_export new file mode 100644 index 00000000..2dd744d3 --- /dev/null +++ b/tests/Wallabag/ImportBundle/fixtures/pinboard_export @@ -0,0 +1,5 @@ +[{"href":"https:\/\/developers.google.com\/web\/updates\/2016\/07\/infinite-scroller","description":"Complexities of an Infinite Scroller","extended":"TL;DR: Re-use your DOM elements and remove the ones that are far away from the viewport. Use placeholders to account for delayed data","meta":"21ff61c6f648901168f9e6119f53df7d","hash":"e69b65724cca1c585b446d4c47865d76","time":"2016-10-31T15:57:56Z","shared":"yes","toread":"no","tags":"infinite dom performance scroll"}, +{"href":"https:\/\/ma.ttias.be\/varnish-explained\/","description":"Varnish (explained) for PHP developers","extended":"A few months ago, I gave a presentation at LaraconEU in Amsterdam titled \"Varnish for PHP developers\". The generic title of that presentation is actually Varnish Explained and this is a write-up of that presentation, the video and the slides.","meta":"d32ad9fac2ed29da4aec12c562e9afb1","hash":"21dd6bdda8ad62666a2c9e79f6e80f98","time":"2016-10-26T06:43:03Z","shared":"yes","toread":"no","tags":"varnish PHP"}, +{"href":"https:\/\/ilia.ws\/files\/nginx_torontophpug.pdf","description":"Nginx Tricks for PHP Developers","extended":"","meta":"9adbb5c4ca6760e335b920800d88c70a","hash":"0189bb08f8bd0122c6544bed4624c546","time":"2016-10-05T07:11:27Z","shared":"yes","toread":"no","tags":"nginx PHP best_practice"}, +{"href":"https:\/\/jolicode.com\/blog\/starting-a-mobile-application-with-react-native","description":"Starting a mobile application with React Native","extended":"While preparing our next React Native training, I learnt a lot on the library and discovered an amazing community with a lot of packages.","meta":"bd140bd3e53e3a0b4cb08cdaf64bcbfc","hash":"015fa10cd97f56186420555e52cfab62","time":"2016-09-23T10:58:20Z","shared":"yes","toread":"no","tags":"react-native"}, +{"href":"http:\/\/open.blogs.nytimes.com\/2016\/08\/29\/testing-varnish-using-varnishtest\/","description":"Testing Varnish Using Varnishtest","extended":"Varnish ships with the ability to test using the testing tool varnishtest. Varnishtest gives you the ability to write VCL tests you can run on the command line or as part of your build process.","meta":"ca2752a07adea4bab52cd19e8fdbf356","hash":"d3e642cc1274d10e4c12ee31f5dde736","time":"2016-08-30T09:33:24Z","shared":"yes","toread":"no","tags":"varnish test vcl"}] -- 2.41.0