Get rid of global parameter for queue and playlist file handle in options

This commit is contained in:
Ravi 2011-07-22 20:11:10 -04:00
parent 167b0a9025
commit f382bc28d7

View file

@ -49,7 +49,6 @@ std_headers = {
simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii') simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
downloadqueue = Queue.Queue()
def preferredencoding(): def preferredencoding():
"""Get preferred encoding. """Get preferred encoding.
@ -606,10 +605,6 @@ class FileDownloader(object):
# Extract information from URL and process it # Extract information from URL and process it
ie.extract(url) ie.extract(url)
#parallel downloader needs dummy at the end to signal end of queue
#for the thread to exit
for i in xrange(self.params.get('parallel')):
downloadqueue.put({'filename':None })
# Suitable InfoExtractor had been found; go to next URL # Suitable InfoExtractor had been found; go to next URL
break break
@ -617,6 +612,11 @@ class FileDownloader(object):
if not suitable_found: if not suitable_found:
self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url) self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
#parallel downloader needs dummy at the end to signal end of queue
#for the thread to exit
for i in xrange(self.params.get('parallel')):
FileDownloader.downloadqueue.put({'filename':None })
return self._download_retcode return self._download_retcode
def post_process(self, filename, ie_info): def post_process(self, filename, ie_info):
@ -661,11 +661,11 @@ class FileDownloader(object):
return False return False
def _do_download(self, filename, url, player_url): def _do_download(self, filename, url, player_url):
if (self.params.get('playlistfile') != None): if (FileDownloader.playlistfile != None):
self.params.get('playlistfile').write(filename+"\n") FileDownloader.playlistfile.write(filename+"\n")
self.params.get('playlistfile').flush() FileDownloader.playlistfile.flush()
if self.params.get('parallel') > 0: if self.params.get('parallel') > 0:
downloadqueue.put({'filename':filename,'url':url,'player_url':player_url,'params':self.params}) FileDownloader.downloadqueue.put({'filename':filename,'url':url,'player_url':player_url,'params':self.params})
return False return False
else: else:
self._do_real_download(filename, url, player_url) self._do_real_download(filename, url, player_url)
@ -810,12 +810,12 @@ def threadedFileDownloader():
Individual threads are created in main function. Individual threads are created in main function.
""" """
while True: while True:
d = downloadqueue.get() d = FileDownloader.downloadqueue.get()
if (d['filename'] == None): if (d['filename'] is None):
break break
fd=FileDownloader(d['params']) fd=FileDownloader(d['params'])
fd._do_real_download(d['filename'],d['url'],d['player_url']) fd._do_real_download(d['filename'],d['url'],d['player_url'])
downloadqueue.task_done() FileDownloader.downloadqueue.task_done()
class InfoExtractor(object): class InfoExtractor(object):
@ -2953,13 +2953,10 @@ if __name__ == '__main__':
facebook_ie = FacebookIE() facebook_ie = FacebookIE()
generic_ie = GenericIE() generic_ie = GenericIE()
playlistfile = None
if (opts.saveplaylist != None): if (opts.saveplaylist != None):
if(opts.saveplaylist.find(".") == -1 ): FileDownloader.playlistfile = open(opts.saveplaylist, "w")
playlist_filename = opts.saveplaylist + ".m3u"
else: else:
playlist_filename = opts.saveplaylist FileDownloader.playlistfile = None
playlistfile=open(playlist_filename,"w")
# File downloader # File downloader
fd = FileDownloader({ fd = FileDownloader({
@ -2998,7 +2995,6 @@ if __name__ == '__main__':
'nopart': opts.nopart, 'nopart': opts.nopart,
'updatetime': opts.updatetime, 'updatetime': opts.updatetime,
'parallel': opts.parallel, 'parallel': opts.parallel,
'playlistfile': playlistfile
}) })
fd.add_info_extractor(youtube_search_ie) fd.add_info_extractor(youtube_search_ie)
fd.add_info_extractor(youtube_pl_ie) fd.add_info_extractor(youtube_pl_ie)
@ -3030,6 +3026,7 @@ if __name__ == '__main__':
downloadparallel = opts.parallel downloadparallel = opts.parallel
threads = [] threads = []
if downloadparallel > 0: if downloadparallel > 0:
FileDownloader.downloadqueue = Queue.Queue()
for threadcount in xrange(downloadparallel): for threadcount in xrange(downloadparallel):
t = threading.Thread(target=threadedFileDownloader) t = threading.Thread(target=threadedFileDownloader)
t.setDaemon(True) t.setDaemon(True)
@ -3046,8 +3043,11 @@ if __name__ == '__main__':
#wait for download threads to terminate #wait for download threads to terminate
if downloadparallel > 0: if downloadparallel > 0:
while True:
for t in threads: for t in threads:
t.join(2**32) t.join(2**32)
if all(not t.isAlive() for t in threads):
break
# Dump cookie jar if requested # Dump cookie jar if requested
if opts.cookiefile is not None: if opts.cookiefile is not None:
@ -3057,7 +3057,7 @@ if __name__ == '__main__':
sys.exit(u'ERROR: unable to save cookie jar') sys.exit(u'ERROR: unable to save cookie jar')
if ( opts.saveplaylist): if ( opts.saveplaylist):
playlistfile.close() FileDownloader.playlistfile.close()
sys.exit(retcode) sys.exit(retcode)
except DownloadError: except DownloadError: