src/mailman/model/cache.py

   1 # Copyright (C) 2016-2023 by the Free Software Foundation, Inc.
   2 #
   3 # This file is part of GNU Mailman.
   4 #
   5 # GNU Mailman is free software: you can redistribute it and/or modify it under
   6 # the terms of the GNU General Public License as published by the Free
   7 # Software Foundation, either version 3 of the License, or (at your option)
   8 # any later version.
   9 #
  10 # GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
  11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13 # more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along with
  16 # GNU Mailman.  If not, see <https://www.gnu.org/licenses/>.
  17
  18 """Generic file cache."""
  19
  20 import os
  21 import hashlib
  22
  23 from contextlib import ExitStack
  24 from lazr.config import as_timedelta
  25 from mailman.config import config
  26 from mailman.database.model import Model
  27 from mailman.database.transaction import dbconnection
  28 from mailman.database.types import SAUnicode
  29 from mailman.interfaces.cache import ICacheManager
  30 from mailman.utilities.datetime import now
  31 from mailman.utilities.filesystem import safe_remove
  32 from public import public
  33 from sqlalchemy import Boolean, Column, DateTime, Integer
  34 from zope.interface import implementer
  35
  36
  37 class CacheEntry(Model):
  38     __tablename__ = 'file_cache'
  39
  40     id = Column(Integer, primary_key=True)
  41     key = Column(SAUnicode)
  42     file_id = Column(SAUnicode)
  43     is_bytes = Column(Boolean)
  44     created_on = Column(DateTime)
  45     expires_on = Column(DateTime)
  46
  47     @dbconnection
  48     def __init__(self, store, key, file_id, is_bytes, lifetime):
  49         self.key = key
  50         self.file_id = file_id
  51         self.is_bytes = is_bytes
  52         self.created_on = now()
  53         self.expires_on = self.created_on + lifetime
  54
  55     @dbconnection
  56     def update(self, store, is_bytes, lifetime):
  57         self.is_bytes = is_bytes
  58         self.created_on = now()
  59         self.expires_on = self.created_on + lifetime
  60
  61     @property
  62     def is_expired(self):
  63         return self.expires_on <= now()
  64
  65
  66 @public
  67 @implementer(ICacheManager)
  68 class CacheManager:
  69     """Manages a cache of files on the file system."""
  70
  71     @staticmethod
  72     def _id_to_path(file_id):
  73         dir_1 = file_id[0:2]
  74         dir_2 = file_id[2:4]
  75         dir_path = os.path.join(config.CACHE_DIR, dir_1, dir_2)
  76         file_path = os.path.join(dir_path, file_id)
  77         return file_path, dir_path
  78
  79     @staticmethod
  80     def _key_to_file_id(key):
  81         # Calculate the file-id/SHA256 hash.  The key must be a string, even
  82         # though the hash algorithm requires bytes.
  83         hashfood = key.encode('raw-unicode-escape')
  84         # Use the hex digest (a str) for readability.
  85         return hashlib.sha256(hashfood).hexdigest()
  86
  87     def _write_contents(self, file_id, contents, is_bytes):
  88         # Calculate the file system path by taking the SHA1 hash, stripping
  89         # out two levels of directory (to reduce the chance of direntry
  90         # exhaustion on some systems).
  91         file_path, dir_path = self._id_to_path(file_id)
  92         os.makedirs(dir_path, exist_ok=True)
  93         # Open the file on the correct mode and write the contents.
  94         with ExitStack() as resources:
  95             if is_bytes:
  96                 fp = resources.enter_context(open(file_path, 'wb'))
  97             else:
  98                 fp = resources.enter_context(
  99                     open(file_path, 'w', encoding='utf-8'))
 100             fp.write(contents)
 101
 102     @dbconnection
 103     def add(self, store, key, contents, lifetime=None):
 104         """See `ICacheManager`."""
 105         if lifetime is None:
 106             lifetime = as_timedelta(config.mailman.cache_life)
 107         is_bytes = isinstance(contents, bytes)
 108         file_id = self._key_to_file_id(key)
 109         # Is there already an unexpired entry under this id in the database?
 110         # If the entry doesn't exist, create it.  If it overwrite both the
 111         # contents and lifetime.
 112         entry = store.query(CacheEntry).filter(
 113             CacheEntry.key == key).one_or_none()
 114         if entry is None:
 115             entry = CacheEntry(key, file_id, is_bytes, lifetime)
 116             store.add(entry)
 117         else:
 118             entry.update(is_bytes, lifetime)
 119         self._write_contents(file_id, contents, is_bytes)
 120         return file_id
 121
 122     @dbconnection
 123     def get(self, store, key, *, expunge=False):
 124         """See `ICacheManager`."""
 125         entry = store.query(CacheEntry).filter(
 126             CacheEntry.key == key).one_or_none()
 127         if entry is None:
 128             return None
 129         file_path, dir_path = self._id_to_path(entry.file_id)
 130         with ExitStack() as resources:
 131             if os.path.isfile(file_path):
 132                 if entry.is_bytes:
 133                     fp = resources.enter_context(open(file_path, 'rb'))
 134                 else:
 135                     fp = resources.enter_context(
 136                         open(file_path, 'r', encoding='utf-8'))
 137                 contents = fp.read()
 138             else:
 139                 if entry.is_bytes:
 140                     contents = b'Cache content lost'
 141                 else:
 142                     contents = 'Cache content lost'
 143         # Do we expunge the cache file?
 144         if expunge:
 145             store.delete(entry)
 146             safe_remove(file_path)
 147         return contents
 148
 149     @dbconnection
 150     def evict(self, store, key):
 151         """See `ICacheManager`"""
 152         entry = store.query(CacheEntry).filter(
 153             CacheEntry.key == key).one_or_none()
 154         if entry is None:
 155             return
 156         file_path, dir_path = self._id_to_path(entry.file_id)
 157         safe_remove(file_path)
 158         store.delete(entry)
 159
 160     @dbconnection
 161     def evict_expired(self, store):
 162         """See `ICacheManager`."""
 163         # Find all the cache entries which have expired.  We can probably do
 164         # this more efficiently, but for now there probably aren't that many
 165         # cached files.
 166         expired_entries = (store.query(CacheEntry)
 167                            .filter(CacheEntry.expires_on <= now())
 168                            .all())
 169         for entry in expired_entries:
 170             file_path, _ = self._id_to_path(entry.file_id)
 171             safe_remove(file_path)
 172             store.delete(entry)
 173
 174     @dbconnection
 175     def clear(self, store):
 176         # Delete all the entries.  We can probably do this more efficiently,
 177         # but for now there probably aren't that many cached files.
 178         for entry in store.query(CacheEntry):
 179             file_path, dir_path = self._id_to_path(entry.file_id)
 180             safe_remove(file_path)
 181             store.delete(entry)