From 48b70054b83e1c57b65b08d8fcbd3e54eb97e242 Mon Sep 17 00:00:00 2001 From: Pierre Verkest Date: Fri, 14 Mar 2025 00:18:38 +0100 Subject: [PATCH 1/2] =?UTF-8?q?[FIX]=C2=A0queue=5Fjob:=20job=20runner=20op?= =?UTF-8?q?en=20pipe=20that=20are=20never=20closed=20properly?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit this make a lot of open file descriptors that we got the limit while instentiate jobrunner from queue_job_cron_jobrunner in the current channel implementation https://github.com/OCA/queue/pull/750 --- queue_job/jobrunner/runner.py | 10 ++++++ queue_job/tests/test_runner_runner.py | 49 +++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/queue_job/jobrunner/runner.py b/queue_job/jobrunner/runner.py index 025c228c62..d4dcabceb2 100644 --- a/queue_job/jobrunner/runner.py +++ b/queue_job/jobrunner/runner.py @@ -367,6 +367,16 @@ def __init__( self._stop = False self._stop_pipe = os.pipe() + def __del__(self): + try: + os.close(self._stop_pipe[0]) + except OSError: + pass + try: + os.close(self._stop_pipe[1]) + except OSError: + pass + @classmethod def from_environ_or_config(cls): scheme = os.environ.get("ODOO_QUEUE_JOB_SCHEME") or queue_job_config.get( diff --git a/queue_job/tests/test_runner_runner.py b/queue_job/tests/test_runner_runner.py index c6486e27ef..131ce6322d 100644 --- a/queue_job/tests/test_runner_runner.py +++ b/queue_job/tests/test_runner_runner.py @@ -3,8 +3,57 @@ # pylint: disable=odoo-addons-relative-import # we are testing, we want to test as we were an external consumer of the API +import os + +from odoo.tests import BaseCase, tagged + from odoo.addons.queue_job.jobrunner import runner from .common import load_doctests load_tests = load_doctests(runner) + + +@tagged("-at_install", "post_install") +class TestRunner(BaseCase): + @classmethod + def _is_open_file_descriptor(cls, fd): + try: + os.fstat(fd) + return True + except OSError: + return False + + def test_runner_file_descriptor(self): + a_runner = runner.QueueJobRunner.from_environ_or_config() + + read_fd, write_fd = a_runner._stop_pipe + self.assertTrue(self._is_open_file_descriptor(read_fd)) + self.assertTrue(self._is_open_file_descriptor(write_fd)) + + del a_runner + + self.assertFalse(self._is_open_file_descriptor(read_fd)) + self.assertFalse(self._is_open_file_descriptor(write_fd)) + + def test_runner_file_closed_read_descriptor(self): + a_runner = runner.QueueJobRunner.from_environ_or_config() + + read_fd, write_fd = a_runner._stop_pipe + os.close(read_fd) + + del a_runner + + self.assertFalse(self._is_open_file_descriptor(read_fd)) + self.assertFalse(self._is_open_file_descriptor(write_fd)) + + def test_runner_file_closed_write_descriptor(self): + a_runner = runner.QueueJobRunner.from_environ_or_config() + + read_fd, write_fd = a_runner._stop_pipe + os.close(write_fd) + + del a_runner + + self.assertFalse(self._is_open_file_descriptor(read_fd)) + self.assertFalse(self._is_open_file_descriptor(write_fd)) From 1840d7c9423f78ec65de970598e3e32cb9d316c9 Mon Sep 17 00:00:00 2001 From: OCA-git-bot Date: Wed, 19 Mar 2025 14:58:23 +0000 Subject: [PATCH 2/2] [BOT] post-merge updates --- README.md | 2 +- queue_job/README.rst | 2 +- queue_job/__manifest__.py | 2 +- queue_job/static/description/index.html | 52 ++++++++++++------------- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index b93c251f44..497ebc99ea 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ addon | version | maintainers | summary [base_export_async](base_export_async/) | 14.0.1.0.1 | | Asynchronous export with job queue [base_import_async](base_import_async/) | 14.0.1.0.2 | | Import CSV files in the background [export_async_schedule](export_async_schedule/) | 14.0.1.0.1 | [![guewen](https://github.com/guewen.png?size=30px)](https://github.com/guewen) | Generate and send exports by emails on a schedule -[queue_job](queue_job/) | 14.0.3.9.2 | [![guewen](https://github.com/guewen.png?size=30px)](https://github.com/guewen) | Job Queue +[queue_job](queue_job/) | 14.0.3.9.3 | [![guewen](https://github.com/guewen.png?size=30px)](https://github.com/guewen) | Job Queue [queue_job_batch](queue_job_batch/) | 14.0.1.0.2 | | Job Queue Batch [queue_job_context](queue_job_context/) | 14.0.1.0.1 | [![AshishHirapara](https://github.com/AshishHirapara.png?size=30px)](https://github.com/AshishHirapara) | Queue Job, prepare context before enqueue keys [queue_job_cron](queue_job_cron/) | 14.0.2.0.0 | | Scheduled Actions as Queue Jobs diff --git a/queue_job/README.rst b/queue_job/README.rst index 9abe7be9fe..d07f34b59c 100644 --- a/queue_job/README.rst +++ b/queue_job/README.rst @@ -7,7 +7,7 @@ Job Queue !! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - !! source digest: sha256:7548ac634e7444ca933ea5410b86f4c5edef0496361b60dc5232a6a327b74ff0 + !! source digest: sha256:92d72e5fbf867c96e9c7ae0ab9bb7f70a2f793e83ed9bf8389d6e35fe970828a !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! .. |badge1| image:: https://img.shields.io/badge/maturity-Mature-brightgreen.png diff --git a/queue_job/__manifest__.py b/queue_job/__manifest__.py index 2d03027fce..7c8c95586f 100644 --- a/queue_job/__manifest__.py +++ b/queue_job/__manifest__.py @@ -2,7 +2,7 @@ { "name": "Job Queue", - "version": "14.0.3.9.2", + "version": "14.0.3.9.3", "author": "Camptocamp,ACSONE SA/NV,Odoo Community Association (OCA)", "website": "https://github.com/OCA/queue", "license": "LGPL-3", diff --git a/queue_job/static/description/index.html b/queue_job/static/description/index.html index 54ab52616a..89c408906a 100644 --- a/queue_job/static/description/index.html +++ b/queue_job/static/description/index.html @@ -367,7 +367,7 @@

Job Queue

!! This file is generated by oca-gen-addon-readme !! !! changes will be overwritten. !! !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -!! source digest: sha256:7548ac634e7444ca933ea5410b86f4c5edef0496361b60dc5232a6a327b74ff0 +!! source digest: sha256:92d72e5fbf867c96e9c7ae0ab9bb7f70a2f793e83ed9bf8389d6e35fe970828a !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -->

Mature License: LGPL-3 OCA/queue Translate me on Weblate Try me on Runboat

This addon adds an integrated Job Queue to Odoo.

@@ -375,19 +375,19 @@

Job Queue

Jobs are executed in the background by a Jobrunner, in their own transaction.

Example:

-from odoo import models, fields, api
+from odoo import models, fields, api
 
-class MyModel(models.Model):
+class MyModel(models.Model):
    _name = 'my.model'
 
-   def my_method(self, a, k=None):
+   def my_method(self, a, k=None):
        _logger.info('executed with a: %s and k: %s', a, k)
 
 
-class MyOtherModel(models.Model):
+class MyOtherModel(models.Model):
     _name = 'my.other.model'
 
-    def button_do_stuff(self):
+    def button_do_stuff(self):
         self.env['my.model'].with_delay().my_method('a', k=2)

In the snippet of code above, when we call button_do_stuff, a job capturing @@ -506,7 +506,7 @@

Delaying jobs

The fast way to enqueue a job for a method is to use with_delay() on a record or model:

-def button_done(self):
+def button_done(self):
     self.with_delay().print_confirmation_document(self.state)
     self.write({"state": "done"})
     return True
@@ -522,7 +522,7 @@ 
Delaying jobs

 on a record or model. The following is the equivalent of with_delay() but using the
 long form:

 
-def button_done(self):
+def button_done(self):
     delayable = self.delayable()
     delayable.print_confirmation_document(self.state)
     delayable.delay()
@@ -532,7 +532,7 @@ 
Delaying jobs

 
Methods of Delayable objects return itself, so it can be used as a builder pattern,
 which in some cases allow to build the jobs dynamically:

 
-def button_generate_simple_with_delayable(self):
+def button_generate_simple_with_delayable(self):
     self.ensure_one()
     # Introduction of a delayable object, using a builder pattern
     # allowing to chain jobs or set properties. The delay() method
@@ -548,7 +548,7 @@ 
Delaying jobs

 

 
The simplest way to define a dependency is to use .on_done(job) on a Delayable:

 
-def button_chain_done(self):
+def button_chain_done(self):
     self.ensure_one()
     job1 = self.browse(1).delayable().generate_thumbnail((50, 50))
     job2 = self.browse(1).delayable().generate_thumbnail((50, 50))
@@ -565,9 +565,9 @@ 
Delaying jobs

 [B] of jobs. When and only when all the jobs of the group [A] are executed, the
 jobs of the group [B] are executed. The code would look like:

 
-from odoo.addons.queue_job.delay import group, chain
+from odoo.addons.queue_job.delay import group, chain
 
-def button_done(self):
+def button_done(self):
     group_a = group(self.delayable().method_foo(), self.delayable().method_bar())
     group_b = group(self.delayable().method_baz(1), self.delayable().method_baz(2))
     chain(group_a, group_b).delay()
@@ -588,7 +588,7 @@ 
Delaying jobs

 work. This can be useful to avoid very long jobs, parallelize some task and get more specific
 errors. Usage is as follows:

 
-def button_split_delayable(self):
+def button_split_delayable(self):
     (
         self  # Can be a big recordset, let's say 1000 records
         .delayable()
@@ -603,7 +603,7 @@ 
Delaying jobs

 True, the jobs will be chained, meaning that the next job will only start when the previous
 one is done:

 
-def button_increment_var(self):
+def button_increment_var(self):
     (
         self
         .delayable()
@@ -687,10 +687,10 @@ 
Configure default options for job
 
 
Example of related action code:

 
-class QueueJob(models.Model):
+class QueueJob(models.Model):
     _inherit = 'queue.job'
 
-    def related_action_partner(self, name):
+    def related_action_partner(self, name):
         self.ensure_one()
         model = self.model_name
         partner = self.records
@@ -732,12 +732,12 @@ 
Configure default options for job
 be customized in Base._job_prepare_context_before_enqueue_keys.

 
Example:

 
-class Base(models.AbstractModel):
+class Base(models.AbstractModel):
 
     _inherit = "base"
 
     @api.model
-    def _job_prepare_context_before_enqueue_keys(self):
+    def _job_prepare_context_before_enqueue_keys(self):
         """Keys to keep in context of stored jobs
 
         Empty by default for backward compatibility.
@@ -755,7 +755,7 @@ 
Configure default options for job
 
Tip: you can do this at test case level like this

 
 @classmethod
-def setUpClass(cls):
+def setUpClass(cls):
     super().setUpClass()
     cls.env = cls.env(context=dict(
         cls.env.context,
@@ -795,20 +795,20 @@ 
Testing

 
A very small example (more details in tests/common.py):

 
 # code
-def my_job_method(self, name, count):
+def my_job_method(self, name, count):
     self.write({"name": " ".join([name] * count)
 
-def method_to_test(self):
+def method_to_test(self):
     count = self.env["other.model"].search_count([])
     self.with_delay(priority=15).my_job_method("Hi!", count=count)
     return count
 
 # tests
-from odoo.addons.queue_job.tests.common import trap_jobs
+from odoo.addons.queue_job.tests.common import trap_jobs
 
 # first test only check the expected behavior of the method and the proper
 # enqueuing of jobs
-def test_method_to_test(self):
+def test_method_to_test(self):
     with trap_jobs() as trap:
         result = self.env["model"].method_to_test()
         expected_count = 12
@@ -824,7 +824,7 @@ 
Testing

 
 
  # second test to validate the behavior of the job unitarily
- def test_my_job_method(self):
+ def test_my_job_method(self):
      record = self.env["model"].browse(1)
      record.my_job_method("Hi!", count=12)
      self.assertEqual(record.name, "Hi! Hi! Hi! Hi! Hi! Hi! Hi! Hi! Hi! Hi! Hi! Hi!")
@@ -832,7 +832,7 @@ 
Testing

 
If you prefer, you can still test the whole thing in a single test, by calling
 jobs_tester.perform_enqueued_jobs() in your test.

 
-def test_method_to_test(self):
+def test_method_to_test(self):
     with trap_jobs() as trap:
         result = self.env["model"].method_to_test()
         expected_count = 12
@@ -867,7 +867,7 @@ 
Testing

 
Tip: you can do this at test case level like this

 
 @classmethod
-def setUpClass(cls):
+def setUpClass(cls):
     super().setUpClass()
     cls.env = cls.env(context=dict(
         cls.env.context,