amplee.storage.storememcache

43 - def __init__(self, servers, key_generator=None, 44 storage=None, encoding='utf-8'):

45 """ 46 The Memcache storage allows to use memcache as the master 47 storage but can also be used as a proxy to another storage 48 providing therefore a transparent and powerful cache 49 mechanism to an application. 50 51 By default it will look for the cmemcache module and 52 if not available will fallback to the regular 53 memcache module. 54 55 The ``servers`` argument is a list of strings supporting the 56 common format to connect to a memcached server. 57 58 The ``key_generator`` is a callable that takes the arguments 59 provided to the info() method and returns a new string 60 used as the key within the memcached instance. 61 It should therefore be unique. If not provided it will 62 compute the sha (sha256 if available) hashing of the arguments 63 provided to the info() method prefixed by the name 64 of the collection. 65 66 The ``storage``, if provided, must be an instance of 67 another storage. Each operation on the memcache storage 68 will then propagated to the inner cache transparently. 69 70 Note that because the cmemcache module is not entirely 71 thread safe, this storage uses a lock to synchronize 72 the different write operations. Hopefully this should 73 not decrease the capacity of this storage. 74 75 Note also that this storage should not really be used 76 as a standalone as its behavior varies slightly from the 77 other storages. You should always use this as a proxy 78 storage. 79 """ 80 self.storage = storage 81 self.key_maker = key_generator or self._key_maker 82 self.encoding = encoding 83 84 self.lock = threading.Lock() 85 86 if isinstance(servers, str): 87 servers = [servers] 88 89 # cmemcache doesn't do well with unicode so 90 # let's just be safe. 91 svrs = [] 92 for s in servers: 93 if isinstance(s, unicode): 94 s = s.encode(encoding) 95 svrs.append(s) 96 97 self.mc = Client(svrs) 98 99 self._set_keys = {}

100

101 - def _key_maker(self, *args):

102 """Default key generator""" 103 seed = [] 104 for a in args: 105 if isinstance(a, unicode): 106 a = a.encode(self.encoding) 107 seed.append(a) 108 return HASH(''.join(seed))

109

110 - def _update_set(self, set_key, member_key, remove=False):

111 set = self.mc.get(set_key) 112 if not set: 113 set = Set() 114 if not remove: 115 set.add(member_key) 116 else: 117 set.discard(member_key) 118 self.mc.set(set_key, set)

119

120 - def _create_set_keys(self, collection_name):

121 member_set_key = self._key_maker(collection_name, 'member') 122 media_set_key = self._key_maker(collection_name, 'media') 123 self._set_keys[collection_name] = (member_set_key, media_set_key) 124 125 return member_set_key, media_set_key

126

127 - def shutdown(self):

128 """ 129 Shutdowns the memcache connection. 130 """ 131 self.mc.disconnect_all()

132

133 - def create_container(self, collection_name):

134 """ 135 Calls the proxied storage create_container method 136 to perform the operation. 137 """ 138 if not collection_name in self._set_keys: 139 member_set_key, media_set_key = self._create_set_keys(collection_name) 140 else: 141 member_set_key, media_set_key = self._set_keys[collection_name] 142 143 self.mc.set(member_set_key, Set()) 144 self.mc.set(media_set_key, Set()) 145 146 if self.storage: 147 self.storage.create_container(collection_name)

148

149 - def info(self, collection_name, resource_name=None, **kwargs):

150 """Returns a StorageResourceInfo of which the 151 key attribute is a StorageMemcacheResourceInfo instance.""" 152 # args[0] is collection_name 153 key = self.key_maker(resource_name) 154 proxied_info = None 155 if self.storage: 156 proxied_info = self.storage.info(collection_name, resource_name, **kwargs) 157 pi = StorageMemcacheResourceInfo(key, proxied_info) 158 159 if collection_name not in self._set_keys: 160 # We stick them here to avoid useless computation later on 161 member_set_key, media_set_key = self._create_set_keys(collection_name) 162 else: 163 member_set_key, media_set_key = self._set_keys[collection_name] 164 165 pi.member_set_key = member_set_key 166 pi.media_set_key = media_set_key 167 168 return StorageResourceInfo(resource_name, pi, collection_name)

169

170 - def get_content(self, info):

171 """ 172 Returns the content as a string of the resource found at 'info.key'. 173 174 Keyword arguments 175 info -- as returned by info() 176 """ 177 content = None 178 try: 179 self.lock.acquire() 180 if info.key.key: 181 content = self.mc.get(info.key.key) 182 if not content and self.storage: 183 content = self.storage.get_content(info.key.proxied_info) 184 if info.key.key and content: 185 self.mc.add(info.key.key, content) 186 self._update_set(info.key.media_set_key, info.key.key) 187 finally: 188 self.lock.release() 189 190 if not content: 191 raise UnknownResource(info.key.key) 192 193 return content

194

195 - def get_meta_data(self, info):

196 """ 197 Returns the content as a string of the resource found at 'info.key'. 198 If no resource could be found, an IOError is raised. 199 200 Keyword arguments 201 info -- as returned by info() 202 """ 203 content = None 204 try: 205 self.lock.acquire() 206 if info.key.key: 207 content = self.mc.get(info.key.key) 208 if not content and self.storage: 209 content = self.storage.get_meta_data(info.key.proxied_info) 210 if info.key.key and content: 211 self.mc.add(info.key.key, content) 212 self._update_set(info.key.member_set_key, info.key.key) 213 finally: 214 self.lock.release() 215 216 if not content: 217 raise UnknownResource(info.key.key) 218 219 return content

220

221 - def put_content(self, info, content, **kwargs):

222 """ 223 Set the content at 'info.key' of the resource. 224 225 Keyword arguments 226 info -- as returned by info() 227 228 content -- data as a string object or a file object. 229 In the latter case read() MUST return the full content 230 as a byte string. 231 """ 232 if hasattr(content, 'read'): 233 content = content.read() 234 235 try: 236 self.lock.acquire() 237 if info.key: 238 self.mc.set(info.key.key, content) 239 self._update_set(info.key.media_set_key, info.key.key) 240 241 # XXX: should this be within the lock? 242 if self.storage: 243 self.storage.put_content(info.key.proxied_info, content, **kwargs) 244 finally: 245 self.lock.release()

246

247 - def put_meta_data(self, info, content, **kwargs):

248 """ 249 Set the content at 'info.key' of the resource. 250 251 Keyword arguments 252 info -- as returned by info() 253 254 content -- data as a string or a file object. 255 In the latter case read() MUST return the full content 256 as a byte string. 257 """ 258 if hasattr(content, 'read'): 259 content = content.read() 260 261 try: 262 self.lock.acquire() 263 if info.key: 264 self.mc.set(info.key.key, content) 265 self._update_set(info.key.member_set_key, info.key.key) 266 if self.storage: 267 self.storage.put_meta_data(info.key.proxied_info, content, **kwargs) 268 finally: 269 self.lock.release()

270

271 - def remove_content(self, info):

272 """ 273 Remove the resource at 'info.key' from the cache 274 and if provided from the proxied storage. 275 276 Keyword arguments 277 info -- as returned by info() 278 """ 279 try: 280 self.lock.acquire() 281 if info.key.key: 282 self._update_set(info.key.media_set_key, info.key.key, remove=True) 283 self.mc.delete(info.key.key) 284 if self.storage: 285 self.storage.remove_content(info.key.proxied_info) 286 finally: 287 self.lock.release()

288

289 - def remove_meta_data(self, info):

290 """ 291 Remove the resource at 'info.key' from the cache 292 and if provided from the proxied storage. 293 294 Keyword arguments 295 info -- as returned by info() 296 """ 297 try: 298 self.lock.acquire() 299 if info.key.key: 300 self._update_set(info.key.member_set_key, info.key.key, remove=True) 301 self.mc.delete(info.key.key) 302 if self.storage: 303 self.storage.remove_meta_data(info.key.proxied_info) 304 finally: 305 self.lock.release()

306

307 - def persist(self, path_list, *args, **kwargs):

308 """ 309 Does nothing in this storage but calls the 310 persist method of the proxied storage. 311 """ 312 path_list = [info.key.proxied_info for info in path_list] 313 if self.storage: 314 self.storage.persist(path_list, *args, **kwargs)

315

316 - def exists(self, info):

317 """ 318 Returns True if the resource at 'info.key' exists in either 319 the cache or alternatively the proxied storage. 320 321 Returns False otherwise. 322 323 Note that because it first looks at the cache, the 324 resource may not be present in the proxied storage and 325 still return True creating some kind of false 326 positive. 327 328 Keyword arguments 329 info -- as returned by info() 330 """ 331 if isinstance(info.key,

Source Code for Module amplee.storage.storememcache