1 # Copyright (C) 2016-2019 by the Free Software Foundation, Inc.
3 # This file is part of GNU Mailman.
5 # GNU Mailman is free software: you can redistribute it and/or modify it under
6 # the terms of the GNU General Public License as published by the Free
7 # Software Foundation, either version 3 of the License, or (at your option)
10 # GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
11 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
15 # You should have received a copy of the GNU General Public License along with
16 # GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
18 """Generic file cache."""
23 from contextlib
import ExitStack
24 from lazr
.config
import as_timedelta
25 from mailman
.config
import config
26 from mailman
.database
.model
import Model
27 from mailman
.database
.transaction
import dbconnection
28 from mailman
.database
.types
import SAUnicode
29 from mailman
.interfaces
.cache
import ICacheManager
30 from mailman
.utilities
.datetime
import now
31 from public
import public
32 from sqlalchemy
import Boolean
, Column
, DateTime
, Integer
33 from zope
.interface
import implementer
36 class CacheEntry(Model
):
37 __tablename__
= 'file_cache'
39 id = Column(Integer
, primary_key
=True)
40 key
= Column(SAUnicode
)
41 file_id
= Column(SAUnicode
)
42 is_bytes
= Column(Boolean
)
43 created_on
= Column(DateTime
)
44 expires_on
= Column(DateTime
)
47 def __init__(self
, store
, key
, file_id
, is_bytes
, lifetime
):
49 self
.file_id
= file_id
50 self
.is_bytes
= is_bytes
51 self
.created_on
= now()
52 self
.expires_on
= self
.created_on
+ lifetime
55 def update(self
, store
, is_bytes
, lifetime
):
56 self
.is_bytes
= is_bytes
57 self
.created_on
= now()
58 self
.expires_on
= self
.created_on
+ lifetime
62 return self
.expires_on
<= now()
66 @implementer(ICacheManager
)
68 """Manages a cache of files on the file system."""
71 def _id_to_path(file_id
):
74 dir_path
= os
.path
.join(config
.CACHE_DIR
, dir_1
, dir_2
)
75 file_path
= os
.path
.join(dir_path
, file_id
)
76 return file_path
, dir_path
79 def _key_to_file_id(key
):
80 # Calculate the file-id/SHA256 hash. The key must be a string, even
81 # though the hash algorithm requires bytes.
82 hashfood
= key
.encode('raw-unicode-escape')
83 # Use the hex digest (a str) for readability.
84 return hashlib
.sha256(hashfood
).hexdigest()
86 def _write_contents(self
, file_id
, contents
, is_bytes
):
87 # Calculate the file system path by taking the SHA1 hash, stripping
88 # out two levels of directory (to reduce the chance of direntry
89 # exhaustion on some systems).
90 file_path
, dir_path
= self
._id
_to
_path
(file_id
)
91 os
.makedirs(dir_path
, exist_ok
=True)
92 # Open the file on the correct mode and write the contents.
93 with
ExitStack() as resources
:
95 fp
= resources
.enter_context(open(file_path
, 'wb'))
97 fp
= resources
.enter_context(
98 open(file_path
, 'w', encoding
='utf-8'))
102 def add(self
, store
, key
, contents
, lifetime
=None):
103 """See `ICacheManager`."""
105 lifetime
= as_timedelta(config
.mailman
.cache_life
)
106 is_bytes
= isinstance(contents
, bytes
)
107 file_id
= self
._key
_to
_file
_id
(key
)
108 # Is there already an unexpired entry under this id in the database?
109 # If the entry doesn't exist, create it. If it overwrite both the
110 # contents and lifetime.
111 entry
= store
.query(CacheEntry
).filter(
112 CacheEntry
.key
== key
).one_or_none()
114 entry
= CacheEntry(key
, file_id
, is_bytes
, lifetime
)
117 entry
.update(is_bytes
, lifetime
)
118 self
._write
_contents
(file_id
, contents
, is_bytes
)
122 def get(self
, store
, key
, *, expunge
=False):
123 """See `ICacheManager`."""
124 entry
= store
.query(CacheEntry
).filter(
125 CacheEntry
.key
== key
).one_or_none()
128 file_path
, dir_path
= self
._id
_to
_path
(entry
.file_id
)
129 with
ExitStack() as resources
:
131 fp
= resources
.enter_context(open(file_path
, 'rb'))
133 fp
= resources
.enter_context(
134 open(file_path
, 'r', encoding
='utf-8'))
136 # Do we expunge the cache file?
143 def evict(self
, store
, key
):
144 """See `ICacheManager`"""
145 entry
= store
.query(CacheEntry
).filter(
146 CacheEntry
.key
== key
).one_or_none()
149 file_path
, dir_path
= self
._id
_to
_path
(entry
.file_id
)
154 def evict_expired(self
, store
):
155 """See `ICacheManager`."""
156 # Find all the cache entries which have expired. We can probably do
157 # this more efficiently, but for now there probably aren't that many
159 for entry
in store
.query(CacheEntry
):
161 file_path
, dir_path
= self
._id
_to
_path
(entry
.file_id
)
166 def clear(self
, store
):
167 # Delete all the entries. We can probably do this more efficiently,
168 # but for now there probably aren't that many cached files.
169 for entry
in store
.query(CacheEntry
):
170 file_path
, dir_path
= self
._id
_to
_path
(entry
.file_id
)