src/mailman/runners/tests/test_task.py

   1 # Copyright (C) 2012-2023 by the Free Software Foundation, Inc.
   2 #
   3 # This file is part of GNU Mailman.
   4 #
   5 # GNU Mailman is free software: you can redistribute it and/or modify it under
   6 # the terms of the GNU General Public License as published by the Free
   7 # Software Foundation, either version 3 of the License, or (at your option)
   8 # any later version.
   9 #
  10 # GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
  11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13 # more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along with
  16 # GNU Mailman.  If not, see <https://www.gnu.org/licenses/>.
  17
  18 """Tests for the Task runner."""
  19
  20 import os
  21 import unittest
  22
  23 from datetime import timedelta
  24 from lazr.config import as_timedelta
  25 from mailman.app.bounces import PENDABLE_LIFETIME
  26 from mailman.app.lifecycle import create_list
  27 from mailman.app.moderator import hold_message
  28 from mailman.config import config
  29 from mailman.database.transaction import dbconnection
  30 from mailman.interfaces.cache import ICacheManager
  31 from mailman.interfaces.messages import IMessageStore
  32 from mailman.interfaces.pending import IPendable, IPendings
  33 from mailman.interfaces.requests import IListRequests
  34 from mailman.interfaces.workflow import IWorkflowStateManager
  35 from mailman.model.bounce import BounceEvent
  36 from mailman.model.message import Message
  37 from mailman.runners.task import TaskRunner
  38 from mailman.testing.helpers import (
  39     LogFileMark,
  40     make_testable_runner,
  41     specialized_message_from_string as mfs,
  42 )
  43 from mailman.testing.layers import ConfigLayer
  44 from mailman.utilities.datetime import factory
  45 from zope.component import getUtility
  46 from zope.interface import implementer
  47
  48
  49 @implementer(IPendable)
  50 class MyPendable(dict):
  51     PEND_TYPE = 'my pended'
  52
  53
  54 class TestTask(unittest.TestCase):
  55     """Test various aspects of the Task runner."""
  56
  57     layer = ConfigLayer
  58
  59     def setUp(self):
  60         self._pendings = getUtility(IPendings)
  61         self._events = BounceEvent
  62         self._wfmanager = getUtility(IWorkflowStateManager)
  63         self._cachemanager = getUtility(ICacheManager)
  64         self._messages = getUtility(IMessageStore)
  65         self._mlist = create_list('ant@example.com')
  66         self._msg1 = mfs("""\
  67 To: ant@example.com
  68 From: anne@example.com
  69 Subject: A message
  70 Message-ID: <msg1>
  71
  72 first message
  73 """)
  74         self._msg2 = mfs("""\
  75 To: ant@example.com
  76 From: anne@example.com
  77 Subject: A message
  78 Message-ID: <msg2>
  79
  80 second message
  81 """)
  82         self._listrequests = IListRequests(self._mlist)
  83         self._runner = make_testable_runner(TaskRunner)
  84         self._cachemanager.add('cache1', 'xxx1', lifetime=timedelta(days=1))
  85         self._cachemanager.add('cache2', 'xxx2', lifetime=timedelta(days=3))
  86         pendable = MyPendable(id=1)
  87         self._token1 = self._pendings.add(pendable, lifetime=timedelta(days=1))
  88         self._token2 = self._pendings.add(pendable, lifetime=timedelta(days=3))
  89         self._wfmanager.save(self._token1)
  90         self._wfmanager.save(self._token2)
  91         self._requestid1 = hold_message(
  92             self._mlist, self._msg1, reason='Testing')
  93         self._requestid2 = hold_message(
  94             self._mlist, self._msg2, reason='Testing')
  95
  96     def test_task_runner(self):
  97         # Test that the task runner deletes expired cache, pendings and
  98         # associated workflows.
  99         self.assertEqual(self._cachemanager.get('cache1'), 'xxx1')
 100         self.assertEqual(self._cachemanager.get('cache2'), 'xxx2')
 101         self.assertEqual(self._pendings.count(), 4)
 102         self.assertEqual(self._wfmanager.count, 2)
 103         mark = LogFileMark('mailman.task')
 104         factory.fast_forward(days=2)
 105         self._runner.run()
 106         self.assertIsNone(self._cachemanager.get('cache1'))
 107         self.assertEqual(self._cachemanager.get('cache2'), 'xxx2')
 108         self.assertEqual(self._pendings.count(), 3)
 109         pended = self._pendings.confirm(self._token2, expunge=False)
 110         self.assertEqual(pended['type'], 'my pended')
 111         self.assertEqual(list(self._wfmanager.get_all_tokens()),
 112                          [self._token2])
 113         log = mark.read()
 114         self.assertIn('Task runner evicted 1 expired pendings', log)
 115         self.assertIn('Task runner deleted 1 orphaned workflows', log)
 116         self.assertIn('Task runner deleted 0 orphaned requests', log)
 117         self.assertIn('Task runner evicted expired cache entries', log)
 118
 119     def test_task_runner_request(self):
 120         # Test that the task runner deletes orphaned requests.
 121         self.assertEqual(self._pendings.count(), 4)
 122         self.assertEqual(self._listrequests.count, 2)
 123         life = as_timedelta(config.mailman.moderator_request_life)
 124         mark = LogFileMark('mailman.task')
 125         factory.fast_forward(days=life.days+1)
 126         self._runner.run()
 127         self.assertEqual(self._pendings.count(), 0)
 128         self.assertEqual(self._listrequests.count, 0)
 129         log = mark.read()
 130         self.assertIn('Task runner deleted 2 orphaned requests', log)
 131
 132     def test_task_runner_messages(self):
 133         # Test that the task runner deletes orphaned messages from the
 134         # message store.
 135         # Initially, there are 2 messages in the store and 4 pendings.
 136         self.assertEqual(len(list(self._messages.messages)), 2)
 137         self.assertEqual(self._pendings.count(), 4)
 138         # Deleting the first request removes the pending but not the message.
 139         self._listrequests.delete_request(self._requestid1)
 140         self.assertEqual(self._pendings.count(), 3)
 141         self.assertEqual(len(list(self._messages.messages)), 2)
 142         mark = LogFileMark('mailman.task')
 143         self._runner.run()
 144         # Now there's only msg2.
 145         self.assertEqual(len(list(self._messages.messages)), 1)
 146         self.assertIsNotNone(self._messages.get_message_by_id('<msg2>'))
 147         log = mark.read()
 148         self.assertIn('Task runner deleted 1 orphaned messages', log)
 149
 150     @dbconnection
 151     def test_task_runner_message_files(self, store):
 152         # Test that task runner deletes message files with no entry in the
 153         # message store.
 154         def count_files():
 155             """ A helper to count the number of saved message files."""
 156             count = 0
 157             base_dir = config.MESSAGES_DIR
 158             for root, dirs, files in os.walk(base_dir):
 159                 if files:
 160                     count += len(files)
 161             return count
 162         # Initally there are two entries in the message store and two files.
 163         self.assertEqual(len(list(self._messages.messages)), 2)
 164         self.assertEqual(count_files(), 2)
 165         # Delete one of the message store entries leaving one but still two
 166         # files.
 167         row = store.query(Message).filter_by(message_id='<msg1>').first()
 168         store.delete(row)
 169         self.assertEqual(len(list(self._messages.messages)), 1)
 170         self.assertEqual(count_files(), 2)
 171         mark = LogFileMark('mailman.task')
 172         self._runner.run()
 173         # Now there's only one file.
 174         self.assertEqual(len(list(self._messages.messages)), 1)
 175         self.assertEqual(count_files(), 1)
 176         log = mark.read()
 177         self.assertIn('Task runner deleted 1 orphaned messages', log)
 178
 179     @dbconnection
 180     def test_task_runner_bounce_events_old_unprocessed(self, store):
 181         # Test that the task runner deletes processed bounce events older than
 182         # PENDABLE_LIFETIME, but not newer ones or unprocessed ones.
 183         # Set one old but unprocessed.
 184         event = self._events(self._mlist.list_id,
 185                              'anne@example.com',
 186                              self._msg1,
 187                              )
 188         event.timestamp -= as_timedelta(PENDABLE_LIFETIME) + as_timedelta('1d')
 189         store.add(event)
 190         mark = LogFileMark('mailman.task')
 191         self._runner.run()
 192         log = mark.read()
 193         self.assertIn('Task runner evicted 0 expired bounce events', log)
 194
 195     @dbconnection
 196     def test_task_runner_bounce_events_old_processed(self, store):
 197         # Test that the task runner deletes processed bounce events older than
 198         # PENDABLE_LIFETIME, but not newer ones or unprocessed ones.
 199         # Set one old and processed.
 200         event = self._events(self._mlist.list_id,
 201                              'anne@example.com',
 202                              self._msg1,
 203                              )
 204         event.timestamp -= as_timedelta(PENDABLE_LIFETIME) + as_timedelta('1d')
 205         event.processed = True
 206         store.add(event)
 207         mark = LogFileMark('mailman.task')
 208         self._runner.run()
 209         log = mark.read()
 210         self.assertIn('Task runner evicted 1 expired bounce events', log)
 211
 212     @dbconnection
 213     def test_task_runner_bounce_events_processed(self, store):
 214         # Test that the task runner deletes processed bounce events older than
 215         # PENDABLE_LIFETIME, but not newer ones or unprocessed ones.
 216         # Set one processed.
 217         event = self._events(self._mlist.list_id,
 218                              'anne@example.com',
 219                              self._msg1,
 220                              )
 221         event.processed = True
 222         store.add(event)
 223         mark = LogFileMark('mailman.task')
 224         self._runner.run()
 225         log = mark.read()
 226         self.assertIn('Task runner evicted 0 expired bounce events', log)