Merge pull request #1941 from wallabag/v2-asynchronous-jobs

Use asynchronous jobs for imports

8 files changed, 143 insertions, 8 deletions

diff --git a/src/Wallabag/ImportBundle/Resources/config/rabbit.yml b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml
new file mode 100644
index 00000000..aa049749
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Resources/config/rabbit.yml
@@ -0,0 +1,30 @@
+# RabbitMQ stuff
+services:
+    wallabag_import.consumer.amqp.pocket:
+        class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.pocket.import"
+            - "@logger"
+    wallabag_import.consumer.amqp.readability:
+        class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.readability.import"
+            - "@logger"
+    wallabag_import.consumer.amqp.wallabag_v1:
+        class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.wallabag_v1.import"
+            - "@logger"
+    wallabag_import.consumer.amqp.wallabag_v2:
+        class: Wallabag\ImportBundle\Consumer\AMQPEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.wallabag_v2.import"
+            - "@logger"
diff --git a/src/Wallabag/ImportBundle/Resources/config/redis.yml b/src/Wallabag/ImportBundle/Resources/config/redis.yml
new file mode 100644
index 00000000..7d3248e5
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Resources/config/redis.yml
@@ -0,0 +1,81 @@
+# Redis stuff
+services:
+    # readability
+    wallabag_import.queue.redis.readability:
+        class: Simpleue\Queue\RedisQueue
+        arguments:
+            - "@wallabag_core.redis.client"
+            - "wallabag.import.readability"
+
+    wallabag_import.producer.redis.readability:
+        class: Wallabag\ImportBundle\Redis\Producer
+        arguments:
+            - "@wallabag_import.queue.redis.readability"
+
+    wallabag_import.consumer.redis.readability:
+        class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.readability.import"
+            - "@logger"
+
+    # pocket
+    wallabag_import.queue.redis.pocket:
+        class: Simpleue\Queue\RedisQueue
+        arguments:
+            - "@wallabag_core.redis.client"
+            - "wallabag.import.pocket"
+
+    wallabag_import.producer.redis.pocket:
+        class: Wallabag\ImportBundle\Redis\Producer
+        arguments:
+            - "@wallabag_import.queue.redis.pocket"
+
+    wallabag_import.consumer.redis.pocket:
+        class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.pocket.import"
+            - "@logger"
+
+    # wallabag v1
+    wallabag_import.queue.redis.wallabag_v1:
+        class: Simpleue\Queue\RedisQueue
+        arguments:
+            - "@wallabag_core.redis.client"
+            - "wallabag.import.wallabag_v1"
+
+    wallabag_import.producer.redis.wallabag_v1:
+        class: Wallabag\ImportBundle\Redis\Producer
+        arguments:
+            - "@wallabag_import.queue.redis.wallabag_v1"
+
+    wallabag_import.consumer.redis.wallabag_v1:
+        class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.wallabag_v1.import"
+            - "@logger"
+
+    # wallabag v2
+    wallabag_import.queue.redis.wallabag_v2:
+        class: Simpleue\Queue\RedisQueue
+        arguments:
+            - "@wallabag_core.redis.client"
+            - "wallabag.import.wallabag_v2"
+
+    wallabag_import.producer.redis.wallabag_v2:
+        class: Wallabag\ImportBundle\Redis\Producer
+        arguments:
+            - "@wallabag_import.queue.redis.wallabag_v2"
+
+    wallabag_import.consumer.redis.wallabag_v2:
+        class: Wallabag\ImportBundle\Consumer\RedisEntryConsumer
+        arguments:
+            - "@doctrine.orm.entity_manager"
+            - "@wallabag_user.user_repository"
+            - "@wallabag_import.wallabag_v2.import"
+            - "@logger"
diff --git a/src/Wallabag/ImportBundle/Resources/config/services.yml b/src/Wallabag/ImportBundle/Resources/config/services.yml
index 520d43af..f03404ae 100644
--- a/src/Wallabag/ImportBundle/Resources/config/services.yml
+++ b/src/Wallabag/ImportBundle/Resources/config/services.yml
@@ -1,3 +1,7 @@
+imports:
+    - { resource: rabbit.yml }
+    - { resource: redis.yml }
+
 services:
     wallabag_import.chain:
         class: Wallabag\ImportBundle\Import\ImportChain
@@ -14,7 +18,6 @@ services:
     wallabag_import.pocket.import:
         class: Wallabag\ImportBundle\Import\PocketImport
         arguments:
-            - "@security.token_storage"
             - "@doctrine.orm.entity_manager"
             - "@wallabag_core.content_proxy"
             - "@craue_config"
diff --git a/src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig b/src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig
new file mode 100644
index 00000000..2390a41f
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Resources/views/Import/_workerEnabled.html.twig
@@ -0,0 +1,8 @@
+{% set redis = craue_setting('import_with_redis') %}
+{% set rabbit = craue_setting('import_with_rabbitmq') %}
+
+{% if redis or rabbit %}
+    <div class="card-panel yellow darken-1 black-text">
+        {{ 'import.worker.enabled'|trans }} <strong>{% if rabbit %}RabbitMQ{% elseif redis %}Redis{% endif %}</strong>
+    </div>
+{% endif %}
diff --git a/src/Wallabag/ImportBundle/Resources/views/Import/check_queue.html.twig b/src/Wallabag/ImportBundle/Resources/views/Import/check_queue.html.twig
new file mode 100644
index 00000000..7168ea35
--- /dev/null
+++ b/src/Wallabag/ImportBundle/Resources/views/Import/check_queue.html.twig
@@ -0,0 +1,11 @@
+{% if nbRedisMessages > 0 %}
+    <script>
+        Materialize.toast('Messages in queue: {{ nbRedisMessages }}', 4000);
+    </script>
+{% endif %}
+
+{% if nbRabbitMessages > 0 %}
+    <script>
+        Materialize.toast('Messages in queue: {{ nbRabbitMessages }}', 4000);
+    </script>
+{% endif %}
diff --git a/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig
index 401dbd33..6195fa07 100644
--- a/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig
+++ b/src/Wallabag/ImportBundle/Resources/views/Pocket/index.html.twig
@@ -6,15 +6,13 @@
 <div class="row">
     <div class="col s12">
         <div class="card-panel settings">
+            {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
+
             {% if not has_consumer_key %}
-                <div class="card-panel red darken-1">
+                <div class="card-panel red white-text">
                     {{ 'import.pocket.config_missing.description'|trans }}
 
-                    {% if is_granted('ROLE_SUPER_ADMIN') %}
-                        {{ 'import.pocket.config_missing.admin_message'|trans({'%keyurls%': '<a href="' ~ path('craue_config_settings_modify') ~ '#set-import">', '%keyurle%':'</a>'})|raw }}
-                    {% else %}
-                        {{ 'import.pocket.config_missing.user_message'|trans }}
-                    {% endif %}
+                    {{ 'import.pocket.config_missing.admin_message'|trans({'%keyurls%': '<a href="' ~ path('config') ~ '">', '%keyurle%':'</a>'})|raw }}
                 </div>
             {% endif %}
 
@@ -29,7 +27,7 @@
                         {{ form_label(form.mark_as_read) }}
                     </div>
                 </div>
-                <button class="btn waves-effect waves-light" type="submit" name="action">
+                <button class="btn waves-effect waves-light" type="submit" name="action" {% if not has_consumer_key %}disabled="disabled"{% endif %}>
                     {{ 'import.pocket.connect_to_pocket'|trans }}
                 </button>
             </form>
diff --git a/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig
index f527d309..74653b0f 100644
--- a/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig
+++ b/src/Wallabag/ImportBundle/Resources/views/Readability/index.html.twig
@@ -6,6 +6,8 @@
 <div class="row">
     <div class="col s12">
         <div class="card-panel settings">
+            {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
+
             <div class="row">
                 <blockquote>{{ import.description|trans }}</blockquote>
                 <p>{{ 'import.readability.how_to'|trans }}</p>
diff --git a/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig b/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig
index 13e24c8c..0b19bc34 100644
--- a/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig
+++ b/src/Wallabag/ImportBundle/Resources/views/WallabagV1/index.html.twig
@@ -6,6 +6,8 @@
 <div class="row">
     <div class="col s12">
         <div class="card-panel settings">
+            {% include 'WallabagImportBundle:Import:_workerEnabled.html.twig' %}
+
             <div class="row">
                 <blockquote>{{ import.description|trans }}</blockquote>
                 <p>{{ 'import.wallabag_v1.how_to'|trans }}</p>