Source Code for Module amplee.storage.stores3

  1  # -*- coding: utf-8 -*- 
  2   
  3  __all__ = ['S3Storage'] 
  4   
  5  try: 
  6      from cStringIO import StringIO 
  7  except ImportError: 
  8      from StringIO import StringIO 
  9   
 10  # http://code.google.com/p/boto/ 
 11  import boto 
 12  from boto.s3.key import Key 
 13  from boto.exception import S3ResponseError 
 14       
 15  from amplee.storage import Storage, StorageResourceInfo 
 16  from amplee.utils import generate_uuid, safe_quote 
 17  from amplee.error import UnknownResource 
 18   
 19 -class S3Storage(Storage): 
 20 -    def __init__(self, aws_access_key_id, aws_secret_access_key, 
 21                   unique_prefix, encoding='utf-8', separator='_', 
 22                   aws_key_lookup=None, aws_file_path=None): 
 23          """ 
 24          Amazon S3 storage for amplee. 
 25          http://www.amazon.com/gp/browse.html?node=16427261 
 26   
 27          If the buck does not exist this storage will create it transparently. 
 28   
 29          Keyword arguments 
 30          aws_access_key_id -- Amazon S3 publick key 
 31           
 32          aws_secret_access_key -- Amazon S3 private key 
 33           
 34          unique_prefix -- Unique prefix used for the creation of buckets 
 35           
 36          encoding -- when your content is unicode it has to be encoded before being sent 
 37          to the Amazon servers (default: utf-8) 
 38           
 39          separator -- character used to join the prefix and the bucket 
 40           
 41          aws_key_lookup -- a callback that takes one argument and that returns a tuple 
 42          with the public_key and the secret_key (see below) 
 43           
 44          aws_file_path -- the argument passed to aws_key_lookup is the path 
 45          to the file containing the AWS keys 
 46           
 47          Set it to something really unique and not None! 
 48          """ 
 49          self.unique_prefix = unique_prefix 
 50          if callable(aws_key_lookup): 
 51              aws_access_key_id, aws_secret_access_key = aws_key_lookup(aws_file_path) 
 52          self.s3conn = boto.connect_s3(aws_access_key_id=aws_access_key_id, 
 53                                        aws_secret_access_key=aws_secret_access_key) 
 54          self.encoding = encoding 
 55          self.separator = separator 
 56          self.decode_on_read = False 
 57           
 58 -    def shutdown(self): 
 59          """Disable the boto connection""" 
 60          self.s3conn = None 
 61   
 62 -    def __bn(self, value): 
 63          """Constructs the bucket name""" 
 64          return "%s%s%s" % (self.unique_prefix, self.separator, value) 
 65   
 66 -    def create_container(self, collection_name): 
 67          """ 
 68          Creates and returns a boto bucket instance for that collection name 
 69          prefixed by self.unique_prefix 
 70          """ 
 71          return self.s3conn.create_bucket(self.__bn(collection_name)) 
 72   
 73 -    def info(self, collection_name, resource_name=None): 
 74          """Returns a StorageResourceInfo instance where the 
 75          ``key`` attribute is a boto.s3.key.Key instance.""" 
 76          bucket = self.create_container(collection_name) 
 77          key = Key(bucket) 
 78          if resource_name: 
 79              key.key = safe_quote(resource_name, encoding=self.encoding) 
 80          return StorageResourceInfo(resource_name, key, collection_name) 
 81   
 82 -    def get_content(self, info): 
 83          """ 
 84          Returns the content of the resource at 'info' as a string 
 85   
 86          Keyword argument: 
 87          info -- as returned by info() 
 88          """ 
 89          if info.key: 
 90              try: 
 91                  content = info.key.get_contents_as_string() 
 92                  if self.decode_on_read: 
 93                      return content.decode(self.encoding) 
 94                  return content 
 95              except S3ResponseError: 
 96                  pass 
 97   
 98          raise UnknownResource(info.name) 
 99   
100 -    def get_meta_data(self, info): 
101          """ 
102          Returns the content of the resource at 'info' as a string 
103   
104          Keyword argument: 
105          info -- as returned by info() 
106          """ 
107          if info.key: 
108              try: 
109                  content = info.key.get_contents_as_string() 
110                  if self.decode_on_read: 
111                      return content.decode(self.encoding) 
112                  return content 
113              except S3ResponseError: 
114                  pass 
115               
116          raise UnknownResource(info.name) 
117       
118 -    def put_content(self, info, content, media_type=None, *args, **kwargs): 
119          """ 
120          Updates the content of the resource at 'info' 
121   
122          Keyword argument: 
123          info -- as returned by info() 
124           
125          content -- content as a string or a file object 
126          to be persisted into the bucket. If a file object is 
127          provided it will be closed automatically. 
128           
129           
130          media_type -- mime type of the resource (default:None) 
131          """ 
132          if info.key: 
133              if hasattr(content, 'read'): 
134                  fp = content 
135              else: 
136                  if isinstance(content, unicode): 
137                      content = content.encode(self.encoding) 
138                  fp = StringIO(content) 
139              if not media_type: 
140                  media_type = kwargs.get('media_type', None) 
141              headers = {'Content-Type': media_type} 
142              info.key.set_contents_from_file(fp, headers=headers, replace=True) 
143              fp.close() 
144   
145 -    def put_meta_data(self, info, content, media_type=None, *args, **kwargs): 
146          """ 
147          Updates the content of the resource at 'info' 
148   
149          Keyword argument: 
150          info -- as returned by info() 
151           
152          content -- content as a string or a file object 
153          to be persisted into the bucket. If a file object is 
154          provided it will be closed automatically. 
155           
156          media_type -- mime type of the resource (default:None) 
157          """ 
158          if info.key: 
159              if hasattr(content, 'read'): 
160                  fp = content 
161              else: 
162                  if isinstance(content, unicode): 
163                      content = content.encode(self.encoding) 
164                  fp = StringIO(content) 
165              if not media_type: 
166                  media_type = kwargs.get('media_type', None) 
167              headers = {'Content-Type': media_type} 
168              info.key.set_contents_from_file(fp, headers=headers, replace=True) 
169              fp.close() 
170   
171 -    def remove_content(self, info): 
172          """ 
173          Removes the resource at 'info' 
174           
175          Keyword argument: 
176          info -- as returned by info() 
177          """ 
178          if info.key: 
179              info.key.bucket.delete_key(info.key) 
180   
181 -    def remove_meta_data(self, info): 
182          """ 
183          Removes the resource at 'info' 
184           
185          Keyword argument: 
186          info -- as returned by info() 
187          """ 
188          if info.key: 
189              info.key.bucket.delete_key(info.key) 
190   
191 -    def persist(self, info_list, msg=None): 
192          """ 
193          Does nothing 
194          """ 
195          pass 
196       
197 -    def exists(self, info): 
198          """ 
199          Returns True if the resource 'info.key' exists in the bucket 
200          """ 
201          if info.key: 
202              result = info.key.bucket.lookup(info.key.key) 
203              if result: 
204                  return True 
205          return False 
206   
207 -    def ls(self, collection_name, ext=None): 
208          """ 
209          List all the members in a collection and returns them as 
210          a dictionnary. 
211           
212          Keyword argument: 
213          collection_name -- the name of the collection to browse 
214          """ 
215          bucket = self.create_container(collection_name) 
216          results = bucket.get_all_keys() 
217          members = {} 
218          for result in results: 
219              if ext: 
220                  if result.key.endswith(ext): 
221                      members[result.key] = StorageResourceInfo(result.key, result, 
222                                                                collection_name) 
223              else: 
224                  members[result.key] = StorageResourceInfo(result.key, result, 
225                                                            collection_name) 
226                   
227          return members 
228   
229 -    def ils(self, collection_name, ext=None): 
230          """ 
231          Yields members in a collection and returns them as 
232          a tuple. 
233           
234          Keyword argument: 
235          collection_name -- the name of the collection to browse 
236          """ 
237          bucket = self.create_container(collection_name) 
238          results = bucket.get_all_keys() 
239           
240          for result in results: 
241              if ext: 
242                  if result.key.endswith(ext): 
243                      yield result.key, StorageResourceInfo(result.key, result, 
244                                                            collection_name) 
245              else: 
246                  yield result.key, StorageResourceInfo(result.key, result, 
247                                                        collection_name) 
248   
249          raise StopIteration() 
250