From 59201088b4fc13fd361238396f630dabd9bd1990 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Wed, 21 Sep 2016 17:47:47 +0200 Subject: [PATCH] bring chrome and firefox as separate imports --- .../Resources/translations/messages.da.yml | 12 +- .../Resources/translations/messages.de.yml | 12 +- .../Resources/translations/messages.en.yml | 10 +- .../Resources/translations/messages.es.yml | 12 +- .../Resources/translations/messages.fa.yml | 12 +- .../Resources/translations/messages.fr.yml | 10 +- .../Resources/translations/messages.it.yml | 12 +- .../Resources/translations/messages.oc.yml | 12 +- .../Resources/translations/messages.pl.yml | 12 +- .../Resources/translations/messages.ro.yml | 12 +- .../Resources/translations/messages.tr.yml | 12 +- .../ImportBundle/Command/ImportCommand.php | 9 +- .../Controller/BrowserController.php | 23 +- .../Controller/ChromeController.php | 41 +++ .../Controller/FirefoxController.php | 41 +++ .../ImportBundle/Import/BrowserImport.php | 247 +++++++++--------- .../ImportBundle/Import/ChromeImport.php | 71 +++++ .../ImportBundle/Import/FirefoxImport.php | 71 +++++ .../ImportBundle/Resources/config/rabbit.yml | 14 + .../ImportBundle/Resources/config/redis.yml | 40 +++ .../Resources/config/services.yml | 15 +- .../Resources/views/Chrome/index.html.twig | 43 +++ .../{Browser => Firefox}/index.html.twig | 4 +- .../Controller/ChromeControllerTest.php | 149 +++++++++++ ...llerTest.php => FirefoxControllerTest.php} | 96 ++++--- .../Controller/ImportControllerTest.php | 2 +- .../Controller/ReadabilityControllerTest.php | 1 - .../ImportBundle/Import/ChromeImportTest.php | 233 +++++++++++++++++ .../ImportBundle/Import/FirefoxImportTest.php | 233 +++++++++++++++++ 29 files changed, 1237 insertions(+), 224 deletions(-) create mode 100644 src/Wallabag/ImportBundle/Controller/ChromeController.php create mode 100644 src/Wallabag/ImportBundle/Controller/FirefoxController.php create mode 100644 src/Wallabag/ImportBundle/Import/ChromeImport.php create mode 100644 src/Wallabag/ImportBundle/Import/FirefoxImport.php create mode 100644 src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig rename src/Wallabag/ImportBundle/Resources/views/{Browser => Firefox}/index.html.twig (93%) create mode 100644 tests/Wallabag/ImportBundle/Controller/ChromeControllerTest.php rename tests/Wallabag/ImportBundle/Controller/{BrowserControllerTest.php => FirefoxControllerTest.php} (63%) create mode 100644 tests/Wallabag/ImportBundle/Import/ChromeImportTest.php create mode 100644 tests/Wallabag/ImportBundle/Import/FirefoxImportTest.php diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml index e1170b75..9eeb210b 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.da.yml @@ -349,10 +349,14 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" - # browser: - # page_title: 'Import > Browser' - # description: "This importer will import all your Firefox or Chrome bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file.

For Chrome, the location of the file depends on your operating system : Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" - # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # firefox: + # page_title: 'Import > Firefox' + # description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + #chrome: + # page_title: 'Import > Chrome' + # description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: # page_title: 'Developer' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml index b066cd88..a9ec2519 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.de.yml @@ -349,10 +349,14 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" - # browser: - # page_title: 'Import > Browser' - # description: "This importer will import all your Firefox or Chrome bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file.

For Chrome, the location of the file depends on your operating system : Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" - # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # firefox: + # page_title: 'Import > Firefox' + # description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # chrome: + # page_title: 'Import > Chrome' + # description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: page_title: 'Entwickler' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml index afc28776..c0d8656d 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.en.yml @@ -349,9 +349,13 @@ import: how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" - browser: - page_title: 'Import > Browser' - description: "This importer will import all your Firefox or Chrome bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file.

For Chrome, the location of the file depends on your operating system : Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + firefox: + page_title: 'Import > Firefox' + description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + chrome: + page_title: 'Import > Chrome' + description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml index 429e5fed..1d6993db 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.es.yml @@ -349,10 +349,14 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" - # browser: - # page_title: 'Import > Browser' - # description: "This importer will import all your Firefox or Chrome bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file.

For Chrome, the location of the file depends on your operating system : Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" - # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # firefox: + # page_title: 'Import > Firefox' + # description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # chrome: + # page_title: 'Import > Chrome' + # description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: page_title: 'Promotor' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml index 8ee6c135..68272f99 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fa.yml @@ -349,10 +349,14 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" - # browser: - # page_title: 'Import > Browser' - # description: "This importer will import all your Firefox or Chrome bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file.

For Chrome, the location of the file depends on your operating system : Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" - # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # firefox: + # page_title: 'Import > Firefox' + # description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # chrome: + # page_title: 'Import > Chrome' + # description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: # page_title: 'Developer' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml index 5af03e61..b28068b6 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.fr.yml @@ -349,9 +349,13 @@ import: how_to: "Choisissez le fichier de votre export Readability et cliquez sur le bouton ci-dessous pour l'importer." worker: enabled: "Les imports sont asynchrones. Une fois l'import commencé un worker externe traitera les messages un par un. Le service activé est :" - browser: - page_title: 'Import > Navigateur' - description: "Cet outil va vous permettre d'importer tous vos marques-pages de Firefox ou de Google Chrome/Chromium.

Pour Firefox, ouvrez le panneau des marques-pages (Ctrl+Maj+O), puis dans « Importation et sauvegarde », choisissez « Sauvegarde... ». Vous allez récupérer un fichier .json.

Pour Google Chrome, la situation du fichier dépend de votre système d'exploitation : Une fois que vous y êtes, copiez le fichier Bookmarks à un endroit où vous le retrouverez.
Notez que si vous utilisez Chromium à la place de Chrome, vous devez corriger les chemins en conséquence.

" + firefox: + page_title: 'Import > Firefox' + description: "Cet outil va vous permettre d'importer tous vos marques-pages de Firefox.

Pour Firefox, ouvrez le panneau des marques-pages (Ctrl+Maj+O), puis dans « Importation et sauvegarde », choisissez « Sauvegarde... ». Vous allez récupérer un fichier .json.

" + how_to: "Choisissez le fichier de sauvegarde de vos marques-page et cliquez sur le bouton pour l'importer. Soyez avertis que le processus peut prendre un temps assez long car tous les articles doivent être récupérés en ligne." + chrome: + page_title: 'Import > Chrome' + description: "Cet outil va vous permettre d'importer tous vos marques-pages de Google Chrome/Chromium. Pour Google Chrome, la situation du fichier dépend de votre système d'exploitation : Une fois que vous y êtes, copiez le fichier Bookmarks à un endroit où vous le retrouverez.
Notez que si vous utilisez Chromium à la place de Chrome, vous devez corriger les chemins en conséquence.

" how_to: "Choisissez le fichier de sauvegarde de vos marques-page et cliquez sur le bouton pour l'importer. Soyez avertis que le processus peut prendre un temps assez long car tous les articles doivent être récupérés en ligne." developer: diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml index dbf4517c..bd98c8c0 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.it.yml @@ -348,10 +348,14 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" - # browser: - # page_title: 'Import > Browser' - # description: "This importer will import all your Firefox or Chrome bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file.

For Chrome, the location of the file depends on your operating system : Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" - # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched" + # firefox: + # page_title: 'Import > Firefox' + # description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # chrome: + # page_title: 'Import > Chrome' + # description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: page_title: 'Sviluppatori' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml index fa72dff4..6da9ff18 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.oc.yml @@ -349,10 +349,14 @@ import: how_to: "Mercés de seleccionar vòstre Readability fichièr e de clicar sul boton dejós per lo telecargar e l'importar." worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" - # browser: - # page_title: 'Import > Browser' - # description: "This importer will import all your Firefox or Chrome bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file.

For Chrome, the location of the file depends on your operating system : Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" - # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # firefox: + # page_title: 'Import > Firefox' + # description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # chrome: + # page_title: 'Import > Chrome' + # description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: page_title: 'Desvolopador' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml index a477ec67..daa34bc0 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.pl.yml @@ -349,10 +349,14 @@ import: how_to: 'Wybierz swój plik eksportu z Readability i kliknij poniższy przycisk, aby go załadować.' worker: enabled: "Import jest wykonywany asynchronicznie. Od momentu rozpoczęcia importu, zewnętrzna usługa może zajmować się na raz tylko jednym zadaniem. Bieżącą usługą jest:" - # browser: - # page_title: 'Import > Browser' - # description: "This importer will import all your Firefox or Chrome bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file.

For Chrome, the location of the file depends on your operating system : Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" - # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # firefox: + # page_title: 'Import > Firefox' + # description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # chrome: + # page_title: 'Import > Chrome' + # description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: page_title: 'Deweloper' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml index 5d848c2e..067f7878 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.ro.yml @@ -349,10 +349,14 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" - # browser: - # page_title: 'Import > Browser' - # description: "This importer will import all your Firefox or Chrome bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file.

For Chrome, the location of the file depends on your operating system : Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" - # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # firefox: + # page_title: 'Import > Firefox' + # description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # chrome: + # page_title: 'Import > Chrome' + # description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: # page_title: 'Developer' diff --git a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml index 41d169d1..62c69510 100644 --- a/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml +++ b/src/Wallabag/CoreBundle/Resources/translations/messages.tr.yml @@ -349,10 +349,14 @@ import: # how_to: 'Please select your Readability export and click on the below button to upload and import it.' worker: # enabled: "Import is made asynchronously. Once the import task is started, an external worker will handle jobs one at a time. The current service is:" - # browser: - # page_title: 'Import > Browser' - # description: "This importer will import all your Firefox or Chrome bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file.

For Chrome, the location of the file depends on your operating system : Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" - # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # firefox: + # page_title: 'Import > Firefox' + # description: "This importer will import all your Firefox bookmarks.

For Firefox, just go to your bookmarks (Ctrl+Maj+O), then into \"Import and backup\", choose \"Backup...\". You will obtain a .json file." + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." + # chrome: + # page_title: 'Import > Chrome' + # description: "This importer will import all your Chrome bookmarks. The location of the file depends on your operating system :

Once you got there, copy the Bookmarks file someplace you'll find.
Note that if you have Chromium instead of Chrome, you'll have to correct paths accordingly.

" + # how_to: "Please choose the bookmark backup file and click on the button below to import it. Note that the process may take a long time since all articles have to be fetched." developer: # page_title: 'Developer' diff --git a/src/Wallabag/ImportBundle/Command/ImportCommand.php b/src/Wallabag/ImportBundle/Command/ImportCommand.php index ac3d1d92..1df38295 100644 --- a/src/Wallabag/ImportBundle/Command/ImportCommand.php +++ b/src/Wallabag/ImportBundle/Command/ImportCommand.php @@ -17,7 +17,7 @@ class ImportCommand extends ContainerAwareCommand ->setDescription('Import entries from a JSON export from a wallabag v1 instance') ->addArgument('userId', InputArgument::REQUIRED, 'User ID to populate') ->addArgument('filepath', InputArgument::REQUIRED, 'Path to the JSON file') - ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: wallabag v1, v2 or browser', 'v1') + ->addOption('importer', null, InputArgument::OPTIONAL, 'The importer to use: wallabag v1, v2, firefox or chrome', 'v1') ->addOption('markAsRead', null, InputArgument::OPTIONAL, 'Mark all entries as read', false) ; } @@ -44,8 +44,11 @@ class ImportCommand extends ContainerAwareCommand case 'v2': $wallabag = $this->getContainer()->get('wallabag_import.wallabag_v2.import'); break; - case 'browser': - $wallabag = $this->getContainer()->get('wallabag_import.browser.import'); + case 'firefox': + $wallabag = $this->getContainer()->get('wallabag_import.firefox.import'); + break; + case 'chrome': + $wallabag = $this->getContainer()->get('wallabag_import.chrome.import'); break; case 'v1': default: diff --git a/src/Wallabag/ImportBundle/Controller/BrowserController.php b/src/Wallabag/ImportBundle/Controller/BrowserController.php index 3b54a72e..144a4880 100644 --- a/src/Wallabag/ImportBundle/Controller/BrowserController.php +++ b/src/Wallabag/ImportBundle/Controller/BrowserController.php @@ -8,27 +8,21 @@ use Symfony\Component\HttpFoundation\Request; use Symfony\Component\HttpFoundation\Response; use Wallabag\ImportBundle\Form\Type\UploadImportType; -class BrowserController extends Controller +abstract class BrowserController extends Controller { /** * Return the service to handle the import. * * @return \Wallabag\ImportBundle\Import\ImportInterface */ - protected function getImportService() - { - return $this->get('wallabag_import.browser.import'); - } + abstract protected function getImportService(); /** * Return the template used for the form. * * @return string */ - protected function getImportTemplate() - { - return 'WallabagImportBundle:Browser:index.html.twig'; - } + abstract protected function getImportTemplate(); /** * @Route("/browser", name="import_browser") @@ -43,15 +37,15 @@ class BrowserController extends Controller $form->handleRequest($request); $wallabag = $this->getImportService(); + $wallabag->setUser($this->getUser()); if ($form->isValid()) { $file = $form->get('file')->getData(); $markAsRead = $form->get('mark_as_read')->getData(); $name = $this->getUser()->getId().'.json'; - if (in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { + if (null !== $file && in_array($file->getClientMimeType(), $this->getParameter('wallabag_import.allow_mimetypes')) && $file->move($this->getParameter('wallabag_import.resource_dir'), $name)) { $res = $wallabag - ->setUser($this->getUser()) ->setFilepath($this->getParameter('wallabag_import.resource_dir').'/'.$name) ->setMarkAsRead($markAsRead) ->import(); @@ -60,12 +54,17 @@ class BrowserController extends Controller if (true === $res) { $summary = $wallabag->getSummary(); - // TODO : Pluralize these messages $message = $this->get('translator')->trans('flashes.import.notice.summary', [ '%imported%' => $summary['imported'], '%skipped%' => $summary['skipped'], ]); + if (0 < $summary['queued']) { + $message = $this->get('translator')->trans('flashes.import.notice.summary_with_queue', [ + '%queued%' => $summary['queued'], + ]); + } + unlink($this->getParameter('wallabag_import.resource_dir').'/'.$name); } diff --git a/src/Wallabag/ImportBundle/Controller/ChromeController.php b/src/Wallabag/ImportBundle/Controller/ChromeController.php new file mode 100644 index 00000000..e4cc322a --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/ChromeController.php @@ -0,0 +1,41 @@ +get('wallabag_import.chrome.import'); + + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer')); + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v1')); + } + + return $service; + } + + /** + * {@inheritdoc} + */ + protected function getImportTemplate() + { + return 'WallabagImportBundle:Chrome:index.html.twig'; + } + + /** + * @Route("/chrome", name="import_chrome") + */ + public function indexAction(Request $request) + { + return parent::indexAction($request); + } +} diff --git a/src/Wallabag/ImportBundle/Controller/FirefoxController.php b/src/Wallabag/ImportBundle/Controller/FirefoxController.php new file mode 100644 index 00000000..e0dd8214 --- /dev/null +++ b/src/Wallabag/ImportBundle/Controller/FirefoxController.php @@ -0,0 +1,41 @@ +get('wallabag_import.firefox.import'); + + if ($this->get('craue_config')->get('import_with_rabbitmq')) { + $service->setProducer($this->get('old_sound_rabbit_mq.import_wallabag_v1_producer')); + } elseif ($this->get('craue_config')->get('import_with_redis')) { + $service->setProducer($this->get('wallabag_import.producer.redis.wallabag_v1')); + } + + return $service; + } + + /** + * {@inheritdoc} + */ + protected function getImportTemplate() + { + return 'WallabagImportBundle:Firefox:index.html.twig'; + } + + /** + * @Route("/firefox", name="import_firefox") + */ + public function indexAction(Request $request) + { + return parent::indexAction($request); + } +} diff --git a/src/Wallabag/ImportBundle/Import/BrowserImport.php b/src/Wallabag/ImportBundle/Import/BrowserImport.php index e3457196..ef7d6d95 100644 --- a/src/Wallabag/ImportBundle/Import/BrowserImport.php +++ b/src/Wallabag/ImportBundle/Import/BrowserImport.php @@ -9,69 +9,24 @@ use Wallabag\CoreBundle\Entity\Entry; use Wallabag\UserBundle\Entity\User; use Wallabag\CoreBundle\Helper\ContentProxy; -class BrowserImport implements ImportInterface +abstract class BrowserImport extends AbstractImport { - protected $user; - protected $em; - protected $logger; - protected $contentProxy; - protected $skippedEntries = 0; - protected $importedEntries = 0; - protected $totalEntries = 0; protected $filepath; - protected $markAsRead; - private $nbEntries; - - public function __construct(EntityManager $em, ContentProxy $contentProxy) - { - $this->em = $em; - $this->logger = new NullLogger(); - $this->contentProxy = $contentProxy; - } - - public function setLogger(LoggerInterface $logger) - { - $this->logger = $logger; - } - - /** - * We define the user in a custom call because on the import command there is no logged in user. - * So we can't retrieve user from the `security.token_storage` service. - * - * @param User $user - * - * @return $this - */ - public function setUser(User $user) - { - $this->user = $user; - - return $this; - } /** * {@inheritdoc} */ - public function getName() - { - return 'Firefox & Google Chrome'; - } + abstract public function getName(); /** * {@inheritdoc} */ - public function getUrl() - { - return 'import_browser'; - } + abstract public function getUrl(); /** * {@inheritdoc} */ - public function getDescription() - { - return 'import.browser.description'; - } + abstract public function getDescription(); /** * {@inheritdoc} @@ -96,124 +51,162 @@ class BrowserImport implements ImportInterface return false; } - $this->nbEntries = 1; + if ($this->producer) { + $this->parseEntriesForProducer($data); + + return true; + } + $this->parseEntries($data); - $this->em->flush(); return true; } - private function parseEntries($data) + /** + * Set file path to the json file. + * + * @param string $filepath + */ + public function setFilepath($filepath) + { + $this->filepath = $filepath; + + return $this; + } + + /** + * Parse and insert all given entries. + * + * @param $entries + */ + protected function parseEntries($entries) { - foreach ($data as $importedEntry) { - $this->parseEntry($importedEntry); + $i = 1; + + foreach ($entries as $importedEntry) { + if ((array) $importedEntry !== $importedEntry) { + continue; + } + + $entry = $this->parseEntry($importedEntry); + + if (null === $entry) { + continue; + } + + // flush every 20 entries + if (($i % 20) === 0) { + $this->em->flush(); + + // clear only affected entities + $this->em->clear(Entry::class); + $this->em->clear(Tag::class); + } + ++$i; } - $this->totalEntries += count($data); + + $this->em->flush(); } - private function parseEntry($importedEntry) + /** + * Parse entries and send them to the queue. + * It should just be a simple loop on all item, no call to the database should be done + * to speedup queuing. + * + * Faster parse entries for Producer. + * We don't care to make check at this time. They'll be done by the consumer. + * + * @param array $entries + */ + protected function parseEntriesForProducer(array $entries) { - if (!is_array($importedEntry)) { - return; + foreach ($entries as $importedEntry) { + + if ((array) $importedEntry !== $importedEntry) { + continue; + } + + // set userId for the producer (it won't know which user is connected) + $importedEntry['userId'] = $this->user->getId(); + + if ($this->markAsRead) { + $importedEntry = $this->setEntryAsRead($importedEntry); + } + + ++$this->queuedEntries; + + $this->producer->publish(json_encode($importedEntry)); } + } - /* Firefox uses guid while Chrome uses id */ + /** + * {@inheritdoc} + */ + public function parseEntry(array $importedEntry) + { if ((!key_exists('guid', $importedEntry) || (!key_exists('id', $importedEntry))) && is_array(reset($importedEntry))) { $this->parseEntries($importedEntry); - return; } + if (key_exists('children', $importedEntry)) { $this->parseEntries($importedEntry['children']); - return; } - if (key_exists('uri', $importedEntry) || key_exists('url', $importedEntry)) { - - /* Firefox uses uri while Chrome uses url */ - $firefox = key_exists('uri', $importedEntry); - - $existingEntry = $this->em - ->getRepository('WallabagCoreBundle:Entry') - ->findByUrlAndUserId(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], $this->user->getId()); - - if (false !== $existingEntry) { - ++$this->skippedEntries; + if (!key_exists('uri', $importedEntry) && !key_exists('url', $importedEntry)) { + return; + } - return; - } + $firefox = key_exists('uri', $importedEntry); - if (false === parse_url(($firefox) ? $importedEntry['uri'] : $importedEntry['url']) || false === filter_var(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], FILTER_VALIDATE_URL)) { - $this->logger->warning('Imported URL '.($firefox) ? $importedEntry['uri'] : $importedEntry['url'].' is not valid'); - ++$this->skippedEntries; + $existingEntry = $this->em + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId(($firefox) ? $importedEntry['uri'] : $importedEntry['url'], $this->user->getId()); - return; - } + if (false !== $existingEntry) { + ++$this->skippedEntries; - try { - $entry = $this->contentProxy->updateEntry( - new Entry($this->user), - ($firefox) ? $importedEntry['uri'] : $importedEntry['url'] - ); - } catch (\Exception $e) { - $this->logger->warning('Error while saving '.($firefox) ? $importedEntry['uri'] : $importedEntry['url']); - ++$this->skippedEntries; + return; + } - return; - } + $data = $this->prepareEntry($importedEntry); - $entry->setArchived($this->markAsRead); + $entry = new Entry($this->user); + $entry->setUrl($data['url']); + $entry->setTitle($data['title']); - $this->em->persist($entry); - ++$this->importedEntries; + // update entry with content (in case fetching failed, the given entry will be return) + $entry = $this->fetchContent($entry, $data['url'], $data); - // flush every 20 entries - if (($this->nbEntries % 20) === 0) { - $this->em->flush(); - $this->em->clear($entry); - } - ++$this->nbEntries; + if (array_key_exists('tags', $data)) { + $this->contentProxy->assignTagsToEntry( + $entry, + $data['tags'] + ); } - } - /** - * Set whether articles must be all marked as read. - * - * @param bool $markAsRead - * - * @return $this - */ - public function setMarkAsRead($markAsRead) - { - $this->markAsRead = $markAsRead; + $entry->setArchived($data['is_archived']); - return $this; - } + if (!empty($data['created_at'])) { + $dt = new \DateTime(); + $entry->setCreatedAt($dt->setTimestamp($data['created_at']/1000)); + } - /** - * Set file path to the json file. - * - * @param string $filepath - * - * @return $this - */ - public function setFilepath($filepath) - { - $this->filepath = $filepath; + $this->em->persist($entry); + ++$this->importedEntries; - return $this; + return $entry; } /** * {@inheritdoc} */ - public function getSummary() + protected function setEntryAsRead(array $importedEntry) { - return [ - 'skipped' => $this->skippedEntries, - 'imported' => $this->importedEntries, - ]; + $importedEntry['is_archived'] = 1; + + return $importedEntry; } } diff --git a/src/Wallabag/ImportBundle/Import/ChromeImport.php b/src/Wallabag/ImportBundle/Import/ChromeImport.php new file mode 100644 index 00000000..7936ee2f --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/ChromeImport.php @@ -0,0 +1,71 @@ + $entry['name'], + 'html' => '', + 'url' => $entry['url'], + 'is_archived' => $this->markAsRead, + 'tags' => '', + 'created_at' => $entry['date_added'], + ]; + + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + + return $data; + } + + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['is_archived'] = 1; + + return $importedEntry; + } +} diff --git a/src/Wallabag/ImportBundle/Import/FirefoxImport.php b/src/Wallabag/ImportBundle/Import/FirefoxImport.php new file mode 100644 index 00000000..cbf10b87 --- /dev/null +++ b/src/Wallabag/ImportBundle/Import/FirefoxImport.php @@ -0,0 +1,71 @@ + $entry['name'], + 'html' => '', + 'url' => $entry['url'], + 'is_archived' => $this->markAsRead, + 'tags' => '', + 'created_at' => $entry['date_added'], + ]; + + if (array_key_exists('tags', $entry) && $entry['tags'] != '') { + $data['tags'] = $entry['tags']; + } + + return $data; + } + + + /** + * {@inheritdoc} + */ + protected function setEntryAsRead(array $importedEntry) + { + $importedEntry['is_archived'] = 1; + + return $importedEntry; + } +} diff --git a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml index aa049749..6ada6302 100644 --- a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml +++ b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml @@ -28,3 +28,17 @@ services: - "@wallabag_user.user_repository" - "@wallabag_import.wallabag_v2.import" - "@logger" + wallabag_import.consumer.amqp.firefox: + class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.firefox.import" + - "@logger" + wallabag_import.consumer.amqp.chrome: + class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.chrome.import" + - "@logger" diff --git a/src/Wallabag/ImportBundle/Resources/config/redis.yml b/src/Wallabag/ImportBundle/Resources/config/redis.yml index 7d3248e5..c47778b8 100644 --- a/src/Wallabag/ImportBundle/Resources/config/redis.yml +++ b/src/Wallabag/ImportBundle/Resources/config/redis.yml @@ -79,3 +79,43 @@ services: - "@wallabag_user.user_repository" - "@wallabag_import.wallabag_v2.import" - "@logger" + + # firefox + wallabag_import.queue.redis.firefox: + class: Simpleue\Queue\RedisQueue + arguments: + - "@wallabag_core.redis.client" + - "wallabag.import.firefox" + + wallabag_import.producer.redis.firefox: + class: Wallabag\ImportBundle\Redis\Producer + arguments: + - "@wallabag_import.queue.redis.firefox" + + wallabag_import.consumer.redis.firefox: + class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.firefox.import" + - "@logger" + + # chrome + wallabag_import.queue.redis.chrome: + class: Simpleue\Queue\RedisQueue + arguments: + - "@wallabag_core.redis.client" + - "wallabag.import.chrome" + + wallabag_import.producer.redis.firefox: + class: Wallabag\ImportBundle\Redis\Producer + arguments: + - "@wallabag_import.queue.redis.chrome" + + wallabag_import.consumer.redis.firefox: + class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_user.user_repository" + - "@wallabag_import.chrome.import" + - "@logger" diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml index d8be5c28..990f336d 100644 --- a/src/Wallabag/ImportBundle/Resources/config/services.yml +++ b/src/Wallabag/ImportBundle/Resources/config/services.yml @@ -57,12 +57,21 @@ services: tags: - { name: wallabag_import.import, alias: readability } - wallabag_import.browser.import: - class: Wallabag\ImportBundle\Import\BrowserImport + wallabag_import.firefox.import: + class: Wallabag\ImportBundle\Import\FirefoxImport arguments: - "@doctrine.orm.entity_manager" - "@wallabag_core.content_proxy" calls: - [ setLogger, [ "@logger" ]] tags: - - { name: wallabag_import.import, alias: browser } + - { name: wallabag_import.import, alias: firefox } + wallabag_import.chrome.import: + class: Wallabag\ImportBundle\Import\ChromeImport + arguments: + - "@doctrine.orm.entity_manager" + - "@wallabag_core.content_proxy" + calls: + - [ setLogger, [ "@logger" ]] + tags: + - { name: wallabag_import.import, alias: chrome } diff --git a/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig new file mode 100644 index 00000000..ead828c6 --- /dev/null +++ b/src/Wallabag/ImportBundle/Resources/views/Chrome/index.html.twig @@ -0,0 +1,43 @@ +{% extends "WallabagCoreBundle::layout.html.twig" %} + +{% block title %}{{ 'import.chrome.page_title'|trans }}{% endblock %} + +{% block content %} +
+
+
+
+
{{ import.description|trans|raw }}
+

{{ 'import.chrome.how_to'|trans }}

+ +
+ {{ form_start(form, {'method': 'POST'}) }} + {{ form_errors(form) }} +
+
+ {{ form_errors(form.file) }} +
+ {{ form.file.vars.label|trans }} + {{ form_widget(form.file) }} +
+
+ +
+
+
+
{{ 'import.form.mark_as_read_title'|trans }}
+ {{ form_widget(form.mark_as_read) }} + {{ form_label(form.mark_as_read) }} +
+
+ + {{ form_widget(form.save, { 'attr': {'class': 'btn waves-effect waves-light'} }) }} + + {{ form_rest(form) }} + +
+
+
+
+
+{% endblock %} diff --git a/src/Wallabag/ImportBundle/Resources/views/Browser/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig similarity index 93% rename from src/Wallabag/ImportBundle/Resources/views/Browser/index.html.twig rename to src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig index bfc74e9d..f975da3f 100644 --- a/src/Wallabag/ImportBundle/Resources/views/Browser/index.html.twig +++ b/src/Wallabag/ImportBundle/Resources/views/Firefox/index.html.twig @@ -1,6 +1,6 @@ {% extends "WallabagCoreBundle::layout.html.twig" %} -{% block title %}{{ 'import.browser.page_title'|trans }}{% endblock %} +{% block title %}{{ 'import.firefox.page_title'|trans }}{% endblock %} {% block content %}
@@ -8,7 +8,7 @@
{{ import.description|trans|raw }}
-

{{ 'import.browser.how_to'|trans }}

+

{{ 'import.firefox.how_to'|trans }}

{{ form_start(form, {'method': 'POST'}) }} diff --git a/tests/Wallabag/ImportBundle/Controller/ChromeControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ChromeControllerTest.php new file mode 100644 index 00000000..448e559f --- /dev/null +++ b/tests/Wallabag/ImportBundle/Controller/ChromeControllerTest.php @@ -0,0 +1,149 @@ +logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/chrome'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + } + + public function testImportChromeWithRabbitEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1); + + $crawler = $client->request('GET', '/import/chrome'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + + $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0); + } + + public function testImportChromeBadFile() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/chrome'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $data = [ + 'upload_import_file[file]' => '', + ]; + + $client->submit($form, $data); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + } + + public function testImportChromeWithRedisEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + $client->getContainer()->get('craue_config')->set('import_with_redis', 1); + + $crawler = $client->request('GET', '/import/chrome'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__.'/../fixtures/chrome-bookmarks', 'Bookmarks'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertEquals(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertContains('flashes.import.notice.summary', $body[0]); + + $this->assertNotEmpty($client->getContainer()->get('wallabag_core.redis.client')->lpop('wallabag.import.chrome')); + + $client->getContainer()->get('craue_config')->set('import_with_redis', 0); + } + + public function testImportWallabagWithChromeFile() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/chrome'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__.'/../fixtures/chrome-bookmarks', 'Bookmarks'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertEquals(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertContains('flashes.import.notice.summary', $body[0]); + + $content = $client->getContainer() + ->get('doctrine.orm.entity_manager') + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId( + 'http://www.usinenouvelle.com/article/la-multiplication-des-chefs-de-projet-est-une-catastrophe-manageriale-majeure-affirme-le-sociologue-francois-dupuy.N307730', + $this->getLoggedInUserId() + ); + + $this->assertEmpty($content->getMimetype()); + $this->assertNotEmpty($content->getPreviewPicture()); + $this->assertNotEmpty($content->getLanguage()); + $this->assertEquals(0, count($content->getTags())); + } + + public function testImportWallabagWithEmptyFile() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $crawler = $client->request('GET', '/import/chrome'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $file = new UploadedFile(__DIR__.'/../fixtures/test.txt', 'test.txt'); + + $data = [ + 'upload_import_file[file]' => $file, + ]; + + $client->submit($form, $data); + + $this->assertEquals(302, $client->getResponse()->getStatusCode()); + + $crawler = $client->followRedirect(); + + $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); + $this->assertContains('flashes.import.notice.failed', $body[0]); + } +} diff --git a/tests/Wallabag/ImportBundle/Controller/BrowserControllerTest.php b/tests/Wallabag/ImportBundle/Controller/FirefoxControllerTest.php similarity index 63% rename from tests/Wallabag/ImportBundle/Controller/BrowserControllerTest.php rename to tests/Wallabag/ImportBundle/Controller/FirefoxControllerTest.php index b686fcd9..2de0aa09 100644 --- a/tests/Wallabag/ImportBundle/Controller/BrowserControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/FirefoxControllerTest.php @@ -5,26 +5,65 @@ namespace Tests\Wallabag\ImportBundle\Controller; use Tests\Wallabag\CoreBundle\WallabagCoreTestCase; use Symfony\Component\HttpFoundation\File\UploadedFile; -class BrowserControllerTest extends WallabagCoreTestCase +class FirefoxControllerTest extends WallabagCoreTestCase { - public function testImportWallabag() + public function testImportFirefox() { $this->logInAs('admin'); $client = $this->getClient(); - $crawler = $client->request('GET', '/import/browser'); + $crawler = $client->request('GET', '/import/firefox'); $this->assertEquals(200, $client->getResponse()->getStatusCode()); $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); } - public function testImportWallabagWithFirefoxFile() + public function testImportFirefoxWithRabbitEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + + $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 1); + + $crawler = $client->request('GET', '/import/firefox'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + + $client->getContainer()->get('craue_config')->set('import_with_rabbitmq', 0); + } + + public function testImportFirefoxBadFile() { $this->logInAs('admin'); $client = $this->getClient(); - $crawler = $client->request('GET', '/import/browser'); + $crawler = $client->request('GET', '/import/firefox'); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); + + $data = [ + 'upload_import_file[file]' => '', + ]; + + $client->submit($form, $data); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + } + + public function testImportFirefoxWithRedisEnabled() + { + $this->logInAs('admin'); + $client = $this->getClient(); + $client->getContainer()->get('craue_config')->set('import_with_redis', 1); + + $crawler = $client->request('GET', '/import/firefox'); + + $this->assertEquals(200, $client->getResponse()->getStatusCode()); + $this->assertEquals(1, $crawler->filter('form[name=upload_import_file] > button[type=submit]')->count()); + $this->assertEquals(1, $crawler->filter('input[type=file]')->count()); + $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); $file = new UploadedFile(__DIR__.'/../fixtures/firefox-bookmarks.json', 'Bookmarks'); @@ -42,41 +81,20 @@ class BrowserControllerTest extends WallabagCoreTestCase $this->assertGreaterThan(1, $body = $crawler->filter('body')->extract(['_text'])); $this->assertContains('flashes.import.notice.summary', $body[0]); - $content = $client->getContainer() - ->get('doctrine.orm.entity_manager') - ->getRepository('WallabagCoreBundle:Entry') - ->findByUrlAndUserId( - 'http://lexpansion.lexpress.fr/high-tech/orange-offre-un-meilleur-reseau-mobile-que-bouygues-et-sfr-free-derriere_1811554.html', - $this->getLoggedInUserId() - ); - - $this->assertNotEmpty($content->getMimetype()); - $this->assertNotEmpty($content->getPreviewPicture()); - $this->assertNotEmpty($content->getLanguage()); - $this->assertEquals(0, count($content->getTags())); - - $content = $client->getContainer() - ->get('doctrine.orm.entity_manager') - ->getRepository('WallabagCoreBundle:Entry') - ->findByUrlAndUserId( - 'http://stackoverflow.com/questions/15017163/parser-for-exported-bookmarks-html-file-of-google-chrome-and-mozilla-in-java', - $this->getLoggedInUserId() - ); + $this->assertNotEmpty($client->getContainer()->get('wallabag_core.redis.client')->lpop('wallabag.import.firefox')); - $this->assertNotEmpty($content->getMimetype()); - $this->assertNotEmpty($content->getPreviewPicture()); - $this->assertEmpty($content->getLanguage()); + $client->getContainer()->get('craue_config')->set('import_with_redis', 0); } - public function testImportWallabagWithChromeFile() + public function testImportWallabagWithFirefoxFile() { $this->logInAs('admin'); $client = $this->getClient(); - $crawler = $client->request('GET', '/import/browser'); + $crawler = $client->request('GET', '/import/firefox'); $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); - $file = new UploadedFile(__DIR__.'/../fixtures/chrome-bookmarks', 'Bookmarks'); + $file = new UploadedFile(__DIR__.'/../fixtures/firefox-bookmarks.json', 'Bookmarks'); $data = [ 'upload_import_file[file]' => $file, @@ -95,7 +113,7 @@ class BrowserControllerTest extends WallabagCoreTestCase ->get('doctrine.orm.entity_manager') ->getRepository('WallabagCoreBundle:Entry') ->findByUrlAndUserId( - 'http://www.usinenouvelle.com/article/la-multiplication-des-chefs-de-projet-est-une-catastrophe-manageriale-majeure-affirme-le-sociologue-francois-dupuy.N307730', + 'http://lexpansion.lexpress.fr/high-tech/orange-offre-un-meilleur-reseau-mobile-que-bouygues-et-sfr-free-derriere_1811554.html', $this->getLoggedInUserId() ); @@ -103,6 +121,18 @@ class BrowserControllerTest extends WallabagCoreTestCase $this->assertNotEmpty($content->getPreviewPicture()); $this->assertNotEmpty($content->getLanguage()); $this->assertEquals(0, count($content->getTags())); + + $content = $client->getContainer() + ->get('doctrine.orm.entity_manager') + ->getRepository('WallabagCoreBundle:Entry') + ->findByUrlAndUserId( + 'http://stackoverflow.com/questions/15017163/parser-for-exported-bookmarks-html-file-of-google-chrome-and-mozilla-in-java', + $this->getLoggedInUserId() + ); + + $this->assertNotEmpty($content->getMimetype()); + $this->assertNotEmpty($content->getPreviewPicture()); + $this->assertEmpty($content->getLanguage()); } public function testImportWallabagWithEmptyFile() @@ -110,7 +140,7 @@ class BrowserControllerTest extends WallabagCoreTestCase $this->logInAs('admin'); $client = $this->getClient(); - $crawler = $client->request('GET', '/import/browser'); + $crawler = $client->request('GET', '/import/firefox'); $form = $crawler->filter('form[name=upload_import_file] > button[type=submit]')->form(); $file = new UploadedFile(__DIR__.'/../fixtures/test.txt', 'test.txt'); diff --git a/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php index 23a7c877..b6783a56 100644 --- a/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/ImportControllerTest.php @@ -24,6 +24,6 @@ class ImportControllerTest extends WallabagCoreTestCase $crawler = $client->request('GET', '/import/'); $this->assertEquals(200, $client->getResponse()->getStatusCode()); - $this->assertEquals(5, $crawler->filter('blockquote')->count()); + $this->assertEquals(6, $crawler->filter('blockquote')->count()); } } diff --git a/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php b/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php index 87ecb9d3..916dd297 100644 --- a/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php +++ b/tests/Wallabag/ImportBundle/Controller/ReadabilityControllerTest.php @@ -57,7 +57,6 @@ class ReadabilityControllerTest extends WallabagCoreTestCase $this->checkRedis(); $this->logInAs('admin'); $client = $this->getClient(); - $client->getContainer()->get('craue_config')->set('import_with_redis', 1); $crawler = $client->request('GET', '/import/readability'); diff --git a/tests/Wallabag/ImportBundle/Import/ChromeImportTest.php b/tests/Wallabag/ImportBundle/Import/ChromeImportTest.php new file mode 100644 index 00000000..f781a4d2 --- /dev/null +++ b/tests/Wallabag/ImportBundle/Import/ChromeImportTest.php @@ -0,0 +1,233 @@ +user = new User(); + + $this->em = $this->getMockBuilder('Doctrine\ORM\EntityManager') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy = $this->getMockBuilder('Wallabag\CoreBundle\Helper\ContentProxy') + ->disableOriginalConstructor() + ->getMock(); + + $wallabag = new ChromeImport($this->em, $this->contentProxy); + + $this->logHandler = new TestHandler(); + $logger = new Logger('test', [$this->logHandler]); + $wallabag->setLogger($logger); + + if (false === $unsetUser) { + $wallabag->setUser($this->user); + } + + return $wallabag; + } + + public function testInit() + { + $chromeImport = $this->getChromeImport(); + + $this->assertEquals('Chrome', $chromeImport->getName()); + $this->assertNotEmpty($chromeImport->getUrl()); + $this->assertEquals('import.chrome.description', $chromeImport->getDescription()); + } + + public function testImport() + { + $chromeImport = $this->getChromeImport(); + $chromeImport->setFilepath(__DIR__.'/../fixtures/chrome-bookmarks'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(4)) + ->method('findByUrlAndUserId') + ->willReturn(false); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->exactly(4)) + ->method('updateEntry') + ->willReturn($entry); + + $res = $chromeImport->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 4, 'queued' => 0], $chromeImport->getSummary()); + } + + public function testImportAndMarkAllAsRead() + { + $chromeImport = $this->getChromeImport(); + $chromeImport->setFilepath(__DIR__.'/../fixtures/readability-read.json'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(2)) + ->method('findByUrlAndUserId') + ->will($this->onConsecutiveCalls(false, true)); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $this->contentProxy + ->expects($this->exactly(1)) + ->method('updateEntry') + ->willReturn(new Entry($this->user)); + + // check that every entry persisted are archived + $this->em + ->expects($this->any()) + ->method('persist') + ->with($this->callback(function ($persistedEntry) { + return $persistedEntry->isArchived(); + })); + + $res = $chromeImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + + $this->assertEquals(['skipped' => 1, 'imported' => 1, 'queued' => 0], $chromeImport->getSummary()); + } + + public function testImportWithRabbit() + { + $chromeImport = $this->getChromeImport(); + $chromeImport->setFilepath(__DIR__.'/../fixtures/readability.json'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer') + ->disableOriginalConstructor() + ->getMock(); + + $producer + ->expects($this->exactly(4)) + ->method('publish'); + + $chromeImport->setProducer($producer); + + $res = $readabilityImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 4], $chromeImport->getSummary()); + } + + public function testImportWithRedis() + { + $chromeImport = $this->getReadabilityImport(); + $chromeImport->setFilepath(__DIR__.'/../fixtures/chrome-bookmarks'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $factory = new RedisMockFactory(); + $redisMock = $factory->getAdapter('Predis\Client', true); + + $queue = new RedisQueue($redisMock, 'chrome'); + $producer = new Producer($queue); + + $chromeImport->setProducer($producer); + + $res = $chromeImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 4], $chromeImport->getSummary()); + + $this->assertNotEmpty($redisMock->lpop('chrome')); + } + + public function testImportBadFile() + { + $chromeImport = $this->getChromeImport(); + $chromeImport->setFilepath(__DIR__.'/../fixtures/wallabag-v1.jsonx'); + + $res = $chromeImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertContains('ChromeImport: unable to read file', $records[0]['message']); + $this->assertEquals('ERROR', $records[0]['level_name']); + } + + public function testImportUserNotDefined() + { + $chromeImport = $this->getChromeImport(true); + $chromeImport->setFilepath(__DIR__.'/../fixtures/readability.json'); + + $res = $chromeImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertContains('ChromeImport: user is not defined', $records[0]['message']); + $this->assertEquals('ERROR', $records[0]['level_name']); + } +} diff --git a/tests/Wallabag/ImportBundle/Import/FirefoxImportTest.php b/tests/Wallabag/ImportBundle/Import/FirefoxImportTest.php new file mode 100644 index 00000000..0b4a28b4 --- /dev/null +++ b/tests/Wallabag/ImportBundle/Import/FirefoxImportTest.php @@ -0,0 +1,233 @@ +user = new User(); + + $this->em = $this->getMockBuilder('Doctrine\ORM\EntityManager') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy = $this->getMockBuilder('Wallabag\CoreBundle\Helper\ContentProxy') + ->disableOriginalConstructor() + ->getMock(); + + $wallabag = new FirefoxImport($this->em, $this->contentProxy); + + $this->logHandler = new TestHandler(); + $logger = new Logger('test', [$this->logHandler]); + $wallabag->setLogger($logger); + + if (false === $unsetUser) { + $wallabag->setUser($this->user); + } + + return $wallabag; + } + + public function testInit() + { + $firefoxImport = $this->getFirefoxImport(); + + $this->assertEquals('Firefox', $firefoxImport->getName()); + $this->assertNotEmpty($firefoxImport->getUrl()); + $this->assertEquals('import.firefox.description', $firefoxImport->getDescription()); + } + + public function testImport() + { + $firefoxImport = $this->getFirefoxImport(); + $firefoxImport->setFilepath(__DIR__.'/../fixtures/firefox-bookmarks.json'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(4)) + ->method('findByUrlAndUserId') + ->willReturn(false); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->exactly(4)) + ->method('updateEntry') + ->willReturn($entry); + + $res = $firefoxImport->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 4, 'queued' => 0], $firefoxImport->getSummary()); + } + + public function testImportAndMarkAllAsRead() + { + $firefoxImport = $this->getFirefoxImport(); + $firefoxImport->setFilepath(__DIR__.'/../fixtures/readability-read.json'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->exactly(2)) + ->method('findByUrlAndUserId') + ->will($this->onConsecutiveCalls(false, true)); + + $this->em + ->expects($this->any()) + ->method('getRepository') + ->willReturn($entryRepo); + + $this->contentProxy + ->expects($this->exactly(1)) + ->method('updateEntry') + ->willReturn(new Entry($this->user)); + + // check that every entry persisted are archived + $this->em + ->expects($this->any()) + ->method('persist') + ->with($this->callback(function ($persistedEntry) { + return $persistedEntry->isArchived(); + })); + + $res = $firefoxImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + + $this->assertEquals(['skipped' => 1, 'imported' => 1, 'queued' => 0], $firefoxImport->getSummary()); + } + + public function testImportWithRabbit() + { + $firefoxImport = $this->getFirefoxImport(); + $firefoxImport->setFilepath(__DIR__.'/../fixtures/readability.json'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $producer = $this->getMockBuilder('OldSound\RabbitMqBundle\RabbitMq\Producer') + ->disableOriginalConstructor() + ->getMock(); + + $producer + ->expects($this->exactly(4)) + ->method('publish'); + + $firefoxImport->setProducer($producer); + + $res = $readabilityImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 4], $firefoxImport->getSummary()); + } + + public function testImportWithRedis() + { + $firefoxImport = $this->getReadabilityImport(); + $firefoxImport->setFilepath(__DIR__.'/../fixtures/firefox-bookmarks.json'); + + $entryRepo = $this->getMockBuilder('Wallabag\CoreBundle\Repository\EntryRepository') + ->disableOriginalConstructor() + ->getMock(); + + $entryRepo->expects($this->never()) + ->method('findByUrlAndUserId'); + + $this->em + ->expects($this->never()) + ->method('getRepository'); + + $entry = $this->getMockBuilder('Wallabag\CoreBundle\Entity\Entry') + ->disableOriginalConstructor() + ->getMock(); + + $this->contentProxy + ->expects($this->never()) + ->method('updateEntry'); + + $factory = new RedisMockFactory(); + $redisMock = $factory->getAdapter('Predis\Client', true); + + $queue = new RedisQueue($redisMock, 'firefox'); + $producer = new Producer($queue); + + $firefoxImport->setProducer($producer); + + $res = $firefoxImport->setMarkAsRead(true)->import(); + + $this->assertTrue($res); + $this->assertEquals(['skipped' => 0, 'imported' => 0, 'queued' => 4], $firefoxImport->getSummary()); + + $this->assertNotEmpty($redisMock->lpop('firefox')); + } + + public function testImportBadFile() + { + $firefoxImport = $this->getFirefoxImport(); + $firefoxImport->setFilepath(__DIR__.'/../fixtures/wallabag-v1.jsonx'); + + $res = $firefoxImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertContains('FirefoxImport: unable to read file', $records[0]['message']); + $this->assertEquals('ERROR', $records[0]['level_name']); + } + + public function testImportUserNotDefined() + { + $firefoxImport = $this->getFirefoxImport(true); + $firefoxImport->setFilepath(__DIR__.'/../fixtures/readability.json'); + + $res = $firefoxImport->import(); + + $this->assertFalse($res); + + $records = $this->logHandler->getRecords(); + $this->assertContains('FirefoxImport: user is not defined', $records[0]['message']); + $this->assertEquals('ERROR', $records[0]['level_name']); + } +} -- 2.41.0