src/mailman/model/cache.py

   1 # Copyright (C) 2016-2019 by the Free Software Foundation, Inc.
   2 #
   3 # This file is part of GNU Mailman.
   4 #
   5 # GNU Mailman is free software: you can redistribute it and/or modify it under
   6 # the terms of the GNU General Public License as published by the Free
   7 # Software Foundation, either version 3 of the License, or (at your option)
   8 # any later version.
   9 #
  10 # GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
  11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12 # FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13 # more details.
  14 #
  15 # You should have received a copy of the GNU General Public License along with
  16 # GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 """Generic file cache."""
  19
  20 import os
  21 import hashlib
  22
  23 from contextlib import ExitStack
  24 from lazr.config import as_timedelta
  25 from mailman.config import config
  26 from mailman.database.model import Model
  27 from mailman.database.transaction import dbconnection
  28 from mailman.database.types import SAUnicode
  29 from mailman.interfaces.cache import ICacheManager
  30 from mailman.utilities.datetime import now
  31 from public import public
  32 from sqlalchemy import Boolean, Column, DateTime, Integer
  33 from zope.interface import implementer
  34
  35
  36 class CacheEntry(Model):
  37     __tablename__ = 'file_cache'
  38
  39     id = Column(Integer, primary_key=True)
  40     key = Column(SAUnicode)
  41     file_id = Column(SAUnicode)
  42     is_bytes = Column(Boolean)
  43     created_on = Column(DateTime)
  44     expires_on = Column(DateTime)
  45
  46     @dbconnection
  47     def __init__(self, store, key, file_id, is_bytes, lifetime):
  48         self.key = key
  49         self.file_id = file_id
  50         self.is_bytes = is_bytes
  51         self.created_on = now()
  52         self.expires_on = self.created_on + lifetime
  53
  54     @dbconnection
  55     def update(self, store, is_bytes, lifetime):
  56         self.is_bytes = is_bytes
  57         self.created_on = now()
  58         self.expires_on = self.created_on + lifetime
  59
  60     @property
  61     def is_expired(self):
  62         return self.expires_on <= now()
  63
  64
  65 @public
  66 @implementer(ICacheManager)
  67 class CacheManager:
  68     """Manages a cache of files on the file system."""
  69
  70     @staticmethod
  71     def _id_to_path(file_id):
  72         dir_1 = file_id[0:2]
  73         dir_2 = file_id[2:4]
  74         dir_path = os.path.join(config.CACHE_DIR, dir_1, dir_2)
  75         file_path = os.path.join(dir_path, file_id)
  76         return file_path, dir_path
  77
  78     @staticmethod
  79     def _key_to_file_id(key):
  80         # Calculate the file-id/SHA256 hash.  The key must be a string, even
  81         # though the hash algorithm requires bytes.
  82         hashfood = key.encode('raw-unicode-escape')
  83         # Use the hex digest (a str) for readability.
  84         return hashlib.sha256(hashfood).hexdigest()
  85
  86     def _write_contents(self, file_id, contents, is_bytes):
  87         # Calculate the file system path by taking the SHA1 hash, stripping
  88         # out two levels of directory (to reduce the chance of direntry
  89         # exhaustion on some systems).
  90         file_path, dir_path = self._id_to_path(file_id)
  91         os.makedirs(dir_path, exist_ok=True)
  92         # Open the file on the correct mode and write the contents.
  93         with ExitStack() as resources:
  94             if is_bytes:
  95                 fp = resources.enter_context(open(file_path, 'wb'))
  96             else:
  97                 fp = resources.enter_context(
  98                     open(file_path, 'w', encoding='utf-8'))
  99             fp.write(contents)
 100
 101     @dbconnection
 102     def add(self, store, key, contents, lifetime=None):
 103         """See `ICacheManager`."""
 104         if lifetime is None:
 105             lifetime = as_timedelta(config.mailman.cache_life)
 106         is_bytes = isinstance(contents, bytes)
 107         file_id = self._key_to_file_id(key)
 108         # Is there already an unexpired entry under this id in the database?
 109         # If the entry doesn't exist, create it.  If it overwrite both the
 110         # contents and lifetime.
 111         entry = store.query(CacheEntry).filter(
 112             CacheEntry.key == key).one_or_none()
 113         if entry is None:
 114             entry = CacheEntry(key, file_id, is_bytes, lifetime)
 115             store.add(entry)
 116         else:
 117             entry.update(is_bytes, lifetime)
 118         self._write_contents(file_id, contents, is_bytes)
 119         return file_id
 120
 121     @dbconnection
 122     def get(self, store, key, *, expunge=False):
 123         """See `ICacheManager`."""
 124         entry = store.query(CacheEntry).filter(
 125             CacheEntry.key == key).one_or_none()
 126         if entry is None:
 127             return None
 128         file_path, dir_path = self._id_to_path(entry.file_id)
 129         with ExitStack() as resources:
 130             if entry.is_bytes:
 131                 fp = resources.enter_context(open(file_path, 'rb'))
 132             else:
 133                 fp = resources.enter_context(
 134                     open(file_path, 'r', encoding='utf-8'))
 135             contents = fp.read()
 136         # Do we expunge the cache file?
 137         if expunge:
 138             store.delete(entry)
 139             os.remove(file_path)
 140         return contents
 141
 142     @dbconnection
 143     def evict(self, store, key):
 144         """See `ICacheManager`"""
 145         entry = store.query(CacheEntry).filter(
 146             CacheEntry.key == key).one_or_none()
 147         if entry is None:
 148             return
 149         file_path, dir_path = self._id_to_path(entry.file_id)
 150         os.remove(file_path)
 151         store.delete(entry)
 152
 153     @dbconnection
 154     def evict_expired(self, store):
 155         """See `ICacheManager`."""
 156         # Find all the cache entries which have expired.  We can probably do
 157         # this more efficiently, but for now there probably aren't that many
 158         # cached files.
 159         for entry in store.query(CacheEntry):
 160             if entry.is_expired:
 161                 file_path, dir_path = self._id_to_path(entry.file_id)
 162                 os.remove(file_path)
 163                 store.delete(entry)
 164
 165     @dbconnection
 166     def clear(self, store):
 167         # Delete all the entries.  We can probably do this more efficiently,
 168         # but for now there probably aren't that many cached files.
 169         for entry in store.query(CacheEntry):
 170             file_path, dir_path = self._id_to_path(entry.file_id)
 171             os.remove(file_path)
 172             store.delete(entry)