Merge pull request #1941 from wallabag/v2-asynchronous-jobs

Use asynchronous jobs for imports

17 files changed, 217 insertions, 11 deletions

diff --git a/app/AppKernel.php b/app/AppKernel.php
index 96e45da8..52f85558 100644
--- a/app/AppKernel.php
+++ b/app/AppKernel.php
@@ -38,6 +38,7 @@ class AppKernel extends Kernel
             new Wallabag\UserBundle\WallabagUserBundle(),
             new Wallabag\ImportBundle\WallabagImportBundle(),
             new Wallabag\AnnotationBundle\WallabagAnnotationBundle(),
+            new OldSound\RabbitMqBundle\OldSoundRabbitMqBundle(),
         ];
 
         if (in_array($this->getEnvironment(), ['dev', 'test'], true)) {
diff --git a/app/DoctrineMigrations/Version20160911214952.php b/app/DoctrineMigrations/Version20160911214952.php
new file mode 100644
index 00000000..35809cec
--- /dev/null
+++ b/app/DoctrineMigrations/Version20160911214952.php
@@ -0,0 +1,42 @@
+<?php
+
+namespace Application\Migrations;
+
+use Doctrine\DBAL\Migrations\AbstractMigration;
+use Doctrine\DBAL\Schema\Schema;
+use Symfony\Component\DependencyInjection\ContainerAwareInterface;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+
+class Version20160911214952 extends AbstractMigration implements ContainerAwareInterface
+{
+    /**
+     * @var ContainerInterface
+     */
+    private $container;
+
+    public function setContainer(ContainerInterface $container = null)
+    {
+        $this->container = $container;
+    }
+
+    private function getTable($tableName)
+    {
+        return $this->container->getParameter('database_table_prefix') . $tableName;
+    }
+
+    /**
+     * @param Schema $schema
+     */
+    public function up(Schema $schema)
+    {
+        $this->addSql('INSERT INTO `'.$this->getTable('craue_config_setting').'` (`name`, `value`, `section`) VALUES (\'import_with_redis\', \'0\', \'import\')');
+        $this->addSql('INSERT INTO `'.$this->getTable('craue_config_setting').'` (`name`, `value`, `section`) VALUES (\'import_with_rabbitmq\', \'0\', \'import\')');
+    }
+
+    /**
+     * @param Schema $schema
+     */
+    public function down(Schema $schema)
+    {
+    }
+}
diff --git a/app/DoctrineMigrations/Version20160916201049.php b/app/DoctrineMigrations/Version20160916201049.php
new file mode 100644
index 00000000..202901e6
--- /dev/null
+++ b/app/DoctrineMigrations/Version20160916201049.php
@@ -0,0 +1,46 @@
+<?php
+
+namespace Application\Migrations;
+
+use Doctrine\DBAL\Migrations\AbstractMigration;
+use Doctrine\DBAL\Schema\Schema;
+use Symfony\Component\DependencyInjection\ContainerAwareInterface;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+
+class Version20160916201049 extends AbstractMigration implements ContainerAwareInterface
+{
+    /**
+     * @var ContainerInterface
+     */
+    private $container;
+
+    public function setContainer(ContainerInterface $container = null)
+    {
+        $this->container = $container;
+    }
+
+    private function getTable($tableName)
+    {
+        return $this->container->getParameter('database_table_prefix') . $tableName;
+    }
+
+    /**
+     * @param Schema $schema
+     */
+    public function up(Schema $schema)
+    {
+        $this->addSql('ALTER TABLE '.$this->getTable('config').' ADD pocket_consumer_key VARCHAR(255) DEFAULT NULL');
+        $this->addSql("DELETE FROM `".$this->getTable('craue_config_setting')."` WHERE `name` = 'pocket_consumer_key';");
+    }
+
+    /**
+     * @param Schema $schema
+     */
+    public function down(Schema $schema)
+    {
+        $this->abortIf($this->connection->getDatabasePlatform()->getName() == 'sqlite', 'Migration can only be executed safely on \'mysql\' or \'postgresql\'.');
+
+        $this->addSql('ALTER TABLE `'.$this->getTable('config').'` DROP pocket_consumer_key');
+        $this->addSql("INSERT INTO `".$this->getTable('craue_config_setting')."` (`name`, `value`, `section`) VALUES ('pocket_consumer_key', NULL, 'import')");
+    }
+}
diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml
index 3478d638..85079330 100644
--- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml
+++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.da.yml
@@ -8,7 +8,8 @@ export_csv: Aktiver eksport til CSV
 export_json: Aktiver eksport til JSON
 export_txt: Aktiver eksport til TXT
 export_xml: Aktiver eksport til XML
-pocket_consumer_key: Brugers nøgle til Pocket for at importere materialer (https://getpocket.com/developer/docs/authentication)
+# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+# import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: Shaarli-URL, hvis tjenesten er aktiv
 share_diaspora: Aktiver deling til Diaspora
 share_mail: Aktiver deling med email
diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml
index f655a27f..eaba14cd 100644
--- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml
+++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.de.yml
@@ -8,7 +8,8 @@ export_csv: CSV-Export aktivieren
 export_json: JSON-Export aktivieren
 export_txt: TXT-Export aktivieren
 export_xml: XML-Export aktivieren
-pocket_consumer_key: Consumer-Key für Pocket, um Inhalte zu importieren (https://getpocket.com/developer/docs/authentication)
+# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+# import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: Shaarli-URL, sofern der Service aktiviert ist
 share_diaspora: Teilen zu Diaspora aktiveren
 share_mail: Teilen via E-Mail aktiveren
diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml
index 48d0ec4b..8aaa27e7 100644
--- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml
+++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.en.yml
@@ -8,7 +8,8 @@ export_csv: Enable CSV export
 export_json: Enable JSON export
 export_txt: Enable TXT export
 export_xml: Enable XML export
-pocket_consumer_key: Consumer key for Pocket to import contents (https://getpocket.com/developer/docs/authentication)
+import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: Shaarli URL, if the service is enabled
 share_diaspora: Enable share to Diaspora
 share_mail: Enable share by email
diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml
index 7aac9adf..bf3a79af 100644
--- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml
+++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.es.yml
@@ -8,7 +8,8 @@ export_csv: Activar exportación a CSV
 export_json: Activar exportación a JSON
 export_txt: Activar exportación a TXT
 export_xml: Activar exportación a XML
-pocket_consumer_key: Consumer key for Pocket to import contents (https://getpocket.com/developer/docs/authentication)
+# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+# import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: Shaarli URL, si el servicio está activado
 share_diaspora: Activar compartir con Diaspora
 share_mail: Activar compartir con email
diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml
index 613cf86d..d86c4343 100644
--- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml
+++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fa.yml
@@ -8,7 +8,8 @@ export_csv:  فعال‌سازی برون‌سپاری به CSV
 export_json:  فعال‌سازی برون‌سپاری به JSON
 export_txt:  فعال‌سازی برون‌سپاری به TXT
 export_xml:  فعال‌سازی برون‌سپاری به XML
-pocket_consumer_key: کلید کاربری Pocket برای درون‌ریزی مطالب (https://getpocket.com/developer/docs/authentication)
+# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+# import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: نشانی Shaarli، اگر فعال بود
 share_diaspora: فعال‌سازی هم‌رسانی به Diaspora
 share_mail:  فعال‌سازی هم‌رسانی با ایمیل
diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml
index f21f2439..5e1ecf26 100644
--- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml
+++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.fr.yml
@@ -8,7 +8,8 @@ export_csv: Activer l'export CSV
 export_json: Activer l'export JSON
 export_txt: Activer l'export TXT
 export_xml: Activer l'export XML
-pocket_consumer_key: Clé d'authentification Pocket pour importer les données (https://getpocket.com/developer/docs/authentication)
+import_with_rabbitmq: Activer RabbitMQ pour gérer les imports de façon asynchrone
+import_with_redis: Activer Redis pour gérer les imports de façon asynchrone
 shaarli_url: URL de Shaarli, si le service Shaarli est activé
 share_diaspora: Activer le partage vers Diaspora
 share_mail: Activer le partage par email
diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml
index 1202edd5..9d820e4b 100644
--- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml
+++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.it.yml
@@ -8,7 +8,8 @@ export_csv: Abilita esportazione CSV
 export_json: Abilita esportazione JSON
 export_txt: Abilita esportazione TXT
 export_xml: Abilita esportazione XML
-pocket_consumer_key: Consumer key per Pocket per importare i contenuti (https://getpocket.com/developer/docs/authentication)
+# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+# import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: Shaarli URL, se il servizio è abilitato
 share_diaspora: Abilita la condivisione con Diaspora
 share_mail: Abilita la condivisione per email
diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml
index dd91ee49..49dc7732 100644
--- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml
+++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.oc.yml
@@ -8,7 +8,8 @@ export_csv: Activar l'expòrt CSV
 export_json: Activar l'expòrt JSON
 export_txt: Activar l'expòrt TXT
 export_xml: Activar l'expòrt XML
-pocket_consumer_key: Clau d'autentificacion Pocket per importar las donadas (https://getpocket.com/developer/docs/authentication)
+# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+# import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: URL de Shaarli, se lo servici Shaarli es activat
 share_diaspora: Activar lo partatge cap a Diaspora
 share_mail: Activar lo partatge per corrièl
diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml
index 93b36c8f..2365f2b3 100644
--- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml
+++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.pl.yml
@@ -8,7 +8,8 @@ export_csv: Włącz eksport do CSV
 export_json: Włącz eksport do JSON
 export_txt: Włącz eksport do TXT
 export_xml: Włącz eksport do XML
-pocket_consumer_key: Klucz klienta Pocket do importu zawartości (https://getpocket.com/developer/docs/authentication)
+# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+# import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: Adress URL Shaarli, jeżeli usługa jest włączona
 share_diaspora: Włącz udostępnianie dla Diaspora
 share_mail: Włącz udostępnianie przez email
diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml
index 326c9f90..20d664f7 100644
--- a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml
+++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.ro.yml
@@ -8,7 +8,8 @@ export_csv: Permite exportare CSV
 export_json: Permite exportare JSON
 export_txt: Permite exportare TXT
 export_xml: Permite exportare XML
-pocket_consumer_key: Cheie consumator pentru importarea contentului din Pocket (https://getpocket.com/developer/docs/authentication)
+# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+# import_with_redis: Enable Redis to import data asynchronously
 shaarli_url: Shaarli URL, dacă serviciul este permis
 share_diaspora: Permite share către Diaspora
 share_mail: Permite share prin email
diff --git a/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml
new file mode 100644
index 00000000..4b4d3edb
--- /dev/null
+++ b/app/Resources/CraueConfigBundle/translations/CraueConfigBundle.tr.yml
@@ -0,0 +1,31 @@
+# download_pictures: Download pictures on your server
+# carrot: Enable share to Carrot
+# diaspora_url: Diaspora URL, if the service is enabled
+# export_epub: Enable ePub export
+# export_mobi: Enable .mobi export
+# export_pdf: Enable PDF export
+# export_csv: Enable CSV export
+# export_json: Enable JSON export
+# export_txt: Enable TXT export
+# export_xml: Enable XML export
+# import_with_rabbitmq: Enable RabbitMQ to import data asynchronously
+# import_with_redis: Enable Redis to import data asynchronously
+# shaarli_url: Shaarli URL, if the service is enabled
+# share_diaspora: Enable share to Diaspora
+# share_mail: Enable share by email
+# share_shaarli: Enable share to Shaarli
+# share_twitter: Enable share to Twitter
+# show_printlink: Display a link to print content
+# wallabag_support_url: Support URL for wallabag
+# wallabag_url: URL of *your* wallabag instance
+# entry: "article"
+# export: "export"
+# import: "import"
+# misc: "misc"
+# modify_settings: "apply"
+# piwik_host: Host of your website in Piwik
+# piwik_site_id: ID of your website in Piwik
+# piwik_enabled: Enable Piwik
+# demo_mode_enabled: "Enable demo mode ? (only used for the wallabag public demo)"
+# demo_mode_username: "Demo user"
+# share_public: Allow public url for entries
diff --git a/app/config/config.yml b/app/config/config.yml
index 31bd8a8c..4b869c4f 100644
--- a/app/config/config.yml
+++ b/app/config/config.yml
@@ -215,3 +215,67 @@ lexik_maintenance:
     response:
         code: 503
         status: "wallabag Service Temporarily Unavailable"
+
+old_sound_rabbit_mq:
+    connections:
+        default:
+            host:     "%rabbitmq_host%"
+            port:     "%rabbitmq_port%"
+            user:     "%rabbitmq_user%"
+            password: "%rabbitmq_password%"
+            vhost:    /
+            lazy:     true
+    producers:
+        import_pocket:
+            connection: default
+            exchange_options:
+                name: 'wallabag.import.pocket'
+                type: topic
+        import_readability:
+            connection: default
+            exchange_options:
+                name: 'wallabag.import.readability'
+                type: topic
+        import_wallabag_v1:
+            connection: default
+            exchange_options:
+                name: 'wallabag.import.wallabag_v1'
+                type: topic
+        import_wallabag_v2:
+            connection: default
+            exchange_options:
+                name: 'wallabag.import.wallabag_v2'
+                type: topic
+    consumers:
+        import_pocket:
+            connection: default
+            exchange_options:
+                name: 'wallabag.import.pocket'
+                type: topic
+            queue_options:
+                name: 'wallabag.import.pocket'
+            callback: wallabag_import.consumer.amqp.pocket
+        import_readability:
+            connection: default
+            exchange_options:
+                name: 'wallabag.import.readability'
+                type: topic
+            queue_options:
+                name: 'wallabag.import.readability'
+            callback: wallabag_import.consumer.amqp.readability
+        import_wallabag_v1:
+            connection: default
+            exchange_options:
+                name: 'wallabag.import.wallabag_v1'
+                type: topic
+            queue_options:
+                name: 'wallabag.import.wallabag_v1'
+            callback: wallabag_import.consumer.amqp.wallabag_v1
+        import_wallabag_v2:
+            connection: default
+            exchange_options:
+                name: 'wallabag.import.wallabag_v2'
+                type: topic
+            queue_options:
+                name: 'wallabag.import.wallabag_v2'
+            callback: wallabag_import.consumer.amqp.wallabag_v2
diff --git a/app/config/parameters.yml.dist b/app/config/parameters.yml.dist
index d092e139..f2e5bec3 100644
--- a/app/config/parameters.yml.dist
+++ b/app/config/parameters.yml.dist
@@ -38,3 +38,15 @@ parameters:
     fosuser_confirmation: true
 
     from_email: no-reply@wallabag.org
+
+    rss_limit: 50
+
+    # RabbitMQ processing
+    rabbitmq_host: localhost
+    rabbitmq_port: 5672
+    rabbitmq_user: guest
+    rabbitmq_password: guest
+
+    # Redis processing
+    redis_host: localhost
+    redis_port: 6379
diff --git a/app/config/parameters_test.yml b/app/config/parameters_test.yml
index 696c95ef..2943b27a 100644
--- a/app/config/parameters_test.yml
+++ b/app/config/parameters_test.yml
@@ -5,4 +5,4 @@ parameters:
     test_database_name: null
     test_database_user: null
     test_database_password: null
-    test_database_path: '%kernel.root_dir%/../data/db/wallabag_testYO.sqlite'
+    test_database_path: '%kernel.root_dir%/../data/db/wallabag_test.sqlite'