Source Code for Module Anomos.Storage

  1  # This program is free software: you can redistribute it and/or modify 
  2  # it under the terms of the GNU General Public License as published by 
  3  # the Free Software Foundation, either version 3 of the License, or 
  4  # (at your option) any later version. 
  5  # 
  6  # This program is distributed in the hope that it will be useful, 
  7  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
  8  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
  9  # GNU General Public License for more details. 
 10  # 
 11  # You should have received a copy of the GNU General Public License 
 12  # along with this program.  If not, see <http://www.gnu.org/licenses/>. 
 13   
 14  # Written by Bram Cohen 
 15   
 16  import os 
 17  from bisect import bisect_right 
 18  from array import array 
 19   
 20  from Anomos import BTFailure 
 21   
 22   
 23 -class FilePool(object): 
 24   
 25 -    def __init__(self, max_files_open): 
 26          self.max_files_open = max_files_open 
 27          self.allfiles = {} 
 28          self.handlebuffer = None 
 29          self.handles = {} 
 30          self.whandles = {} 
 31   
 32 -    def close_all(self): 
 33          failures = {} 
 34          for filename, handle in self.handles.iteritems(): 
 35              try: 
 36                  handle.close() 
 37              except Exception, e: 
 38                  failures[self.allfiles[filename]] = e 
 39          self.handles.clear() 
 40          self.whandles.clear() 
 41          if self.handlebuffer is not None: 
 42              del self.handlebuffer[:] 
 43          for torrent, e in failures.iteritems(): 
 44              torrent.got_exception(e) 
 45   
 46 -    def set_max_files_open(self, max_files_open): 
 47          self.max_files_open = max_files_open 
 48          self.close_all() 
 49          if len(self.allfiles) > self.max_files_open: 
 50              self.handlebuffer = [] 
 51          else: 
 52              self.handlebuffer = None 
 53   
 54 -    def add_files(self, files, torrent): 
 55          for filename in files: 
 56              if filename in self.allfiles: 
 57                  raise BTFailure('File '+filename+' belongs to another running ' 
 58                                  'torrent') 
 59          for filename in files: 
 60              self.allfiles[filename] = torrent 
 61          if self.handlebuffer is None and \ 
 62                 len(self.allfiles) > self.max_files_open: 
 63              self.handlebuffer = self.handles.keys() 
 64   
 65 -    def remove_files(self, files): 
 66          for filename in files: 
 67              del self.allfiles[filename] 
 68          if self.handlebuffer is not None and \ 
 69                 len(self.allfiles) <= self.max_files_open: 
 70              self.handlebuffer = None 
 71   
 72   
 73  # Make this a separate function because having this code in Storage.__init__() 
 74  # would make python print a SyntaxWarning (uses builtin 'file' before 'global') 
 75   
 76 -def bad_libc_workaround(): 
 77      global file 
 78      def file(name, mode = 'r', buffering = None): 
 79          return open(name, mode) 
 80   
 81 -class Storage(object): 
 82   
 83 -    def __init__(self, config, filepool, files, check_only=False): 
 84          self.filepool = filepool 
 85          self.config = config 
 86          self.ranges = [] 
 87          self.myfiles = {} 
 88          self.tops = {} 
 89          self.undownloaded = {} 
 90          self.unallocated = {} 
 91          total = 0 
 92          for filename, length in files: 
 93              self.unallocated[filename] = length 
 94              self.undownloaded[filename] = length 
 95              if length > 0: 
 96                  self.ranges.append((total, total + length, filename)) 
 97                  self.myfiles[filename] = None 
 98              total += length 
 99              if os.path.exists(filename): 
100                  if not os.path.isfile(filename): 
101                      raise BTFailure('File '+filename+' already exists, but ' 
102                                      'is not a regular file') 
103                  l = os.path.getsize(filename) 
104                  if l > length and not check_only: 
105                      h = file(filename, 'rb+') 
106                      h.truncate(length) 
107                      h.close() 
108                      l = length 
109                  self.tops[filename] = l 
110              elif not check_only: 
111                  f = os.path.split(filename)[0] 
112                  if f != '' and not os.path.exists(f): 
113                      os.makedirs(f) 
114                  file(filename, 'wb').close() 
115          self.begins = [i[0] for i in self.ranges] 
116          self.total_length = total 
117          if check_only: 
118              return 
119          self.handles = filepool.handles 
120          self.whandles = filepool.whandles 
121   
122          # Rather implement this as an ugly hack here than change all the 
123          # individual calls. Affects all torrent instances using this module. 
124          if config['enable_bad_libc_workaround']: 
125              bad_libc_workaround() 
126   
127 -    def was_preallocated(self, pos, length): 
128          for filename, begin, end in self._intervals(pos, length): 
129              if self.tops.get(filename, 0) < end: 
130                  return False 
131          return True 
132   
133 -    def get_total_length(self): 
134          return self.total_length 
135   
136 -    def _intervals(self, pos, amount): 
137          r = [] 
138          stop = pos + amount 
139          p = bisect_right(self.begins, pos) - 1 
140          while p < len(self.ranges) and self.ranges[p][0] < stop: 
141              begin, end, filename = self.ranges[p] 
142              r.append((filename, max(pos, begin) - begin, min(end, stop) - begin)) 
143              p += 1 
144          return r 
145   
146 -    def _get_file_handle(self, filename, for_write): 
147          handlebuffer = self.filepool.handlebuffer 
148          if filename in self.handles: 
149              if for_write and filename not in self.whandles: 
150                  self.handles[filename].close() 
151                  self.handles[filename] = file(filename, 'rb+', 0) 
152                  self.whandles[filename] = None 
153              if handlebuffer is not None and handlebuffer[-1] != filename: 
154                  handlebuffer.remove(filename) 
155                  handlebuffer.append(filename) 
156          else: 
157              if for_write: 
158                  self.handles[filename] = file(filename, 'rb+', 0) 
159                  self.whandles[filename] = None 
160              else: 
161                  self.handles[filename] = file(filename, 'rb', 0) 
162              if handlebuffer is not None: 
163                  if len(handlebuffer) >= self.filepool.max_files_open: 
164                      oldfile = handlebuffer.pop(0) 
165                      if oldfile in self.whandles:   # .pop() in python 2.3 
166                          del self.whandles[oldfile] 
167                      self.handles[oldfile].close() 
168                      del self.handles[oldfile] 
169                  handlebuffer.append(filename) 
170          return self.handles[filename] 
171   
172 -    def read(self, pos, amount): 
173          r = [] 
174          for filename, pos, end in self._intervals(pos, amount): 
175              h = self._get_file_handle(filename, False) 
176              h.seek(pos) 
177              r.append(h.read(end - pos)) 
178          r = ''.join(r) 
179          if len(r) != amount: 
180              raise BTFailure('Short read - something truncated files?') 
181          return r 
182   
183 -    def write(self, pos, s): 
184          # might raise an IOError 
185          total = 0 
186          for filename, begin, end in self._intervals(pos, len(s)): 
187              h = self._get_file_handle(filename, True) 
188              h.seek(begin) 
189              h.write(s[total: total + end - begin]) 
190              total += end - begin 
191   
192 -    def close(self): 
193          error = None 
194          for filename in self.handles.keys(): 
195              if filename in self.myfiles: 
196                  try: 
197                      self.handles[filename].close() 
198                  except Exception, e: 
199                      error = e 
200                  del self.handles[filename] 
201                  if filename in self.whandles: 
202                      del self.whandles[filename] 
203          handlebuffer = self.filepool.handlebuffer 
204          if handlebuffer is not None: 
205              handlebuffer = [f for f in handlebuffer if f not in self.myfiles] 
206              self.filepool.handlebuffer = handlebuffer 
207          if error is not None: 
208              raise error 
209   
210 -    def write_fastresume(self, resumefile, amount_done): 
211          resumefile.write('Anomos resume state file, version 1\n') 
212          resumefile.write(str(amount_done) + '\n') 
213          for _, _, filename in self.ranges: 
214              resumefile.write(str(os.path.getsize(filename)) + ' ' + 
215                               str(os.path.getmtime(filename)) + '\n') 
216   
217 -    def check_fastresume(self, resumefile, return_filelist=False, 
218                           piece_size=None, numpieces=None, allfiles=None): 
219          filenames = [name for _, _, name in self.ranges] 
220          if resumefile is not None: 
221              version = resumefile.readline() 
222              if version != 'Anomos resume state file, version 1\n': 
223                  raise BTFailure('Unsupported fastresume file format, ' 
224                        'maybe from another client version') 
225              amount_done = int(resumefile.readline()) 
226          else: 
227              amount_done = size = mtime = 0 
228          for filename in filenames: 
229              if resumefile is not None: 
230                  line = resumefile.readline() 
231                  size, mtime = line.split()[:2] # allow adding extra fields 
232                  size = int(size) 
233                  mtime = int(float(mtime)) 
234              if os.path.exists(filename): 
235                  fsize = os.path.getsize(filename) 
236              else: 
237                  fsize = 0 
238              if fsize > 0 and mtime != os.path.getmtime(filename): 
239                  raise BTFailure("Fastresume info doesn't match file " 
240                                  "modification time") 
241              if size != fsize: 
242                  raise BTFailure("Fastresume data doesn't match actual " 
243                                  "filesize") 
244          if not return_filelist: 
245              return amount_done 
246          if resumefile is None: 
247              return None 
248          if numpieces < 32768: 
249              typecode = 'h' 
250          else: 
251              typecode = 'l' 
252          try: 
253              r = array(typecode) 
254              r.fromfile(resumefile, numpieces) 
255          except Exception, e: 
256              raise BTFailure("Couldn't read fastresume data: " + str(e)) 
257          for i in range(numpieces): 
258              if r[i] >= 0: 
259                  # last piece goes "past the end", doesn't matter 
260                  self.downloaded(r[i] * piece_size, piece_size) 
261              if r[i] != -2: 
262                  self.allocated(i * piece_size, piece_size) 
263          undl = self.undownloaded 
264          unal = self.unallocated 
265          return amount_done, [undl[x] for x in allfiles], \ 
266                 [not unal[x] for x in allfiles] 
267   
268 -    def allocated(self, pos, length): 
269          for filename, begin, end in self._intervals(pos, length): 
270              self.unallocated[filename] -= end - begin 
271   
272 -    def downloaded(self, pos, length): 
273          for filename, begin, end in self._intervals(pos, length): 
274              self.undownloaded[filename] -= end - begin 
275