From d11d05d07acdd11a93b02d750852dea4ae32be3b Mon Sep 17 00:00:00 2001 From: Filippo Valsorda - Campagna Date: Tue, 10 Apr 2012 16:46:36 +0200 Subject: better naming for the sub-modules --- youtube_dl/utils.py | 267 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 267 insertions(+) create mode 100644 youtube_dl/utils.py (limited to 'youtube_dl/utils.py') diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py new file mode 100644 index 000000000..737cca8e1 --- /dev/null +++ b/youtube_dl/utils.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import gzip +import htmlentitydefs +import HTMLParser +import locale +import os +import re +import sys +import zlib +import urllib2 +import email.utils + +try: + import cStringIO as StringIO +except ImportError: + import StringIO + +try: + import json +except ImportError: # Python <2.6, use trivialjson (https://github.com/phihag/trivialjson): + import trivialjson as json + +std_headers = { + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1', + 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Encoding': 'gzip, deflate', + 'Accept-Language': 'en-us,en;q=0.5', +} + +def preferredencoding(): + """Get preferred encoding. + + Returns the best encoding scheme for the system, based on + locale.getpreferredencoding() and some further tweaks. + """ + def yield_preferredencoding(): + try: + pref = locale.getpreferredencoding() + u'TEST'.encode(pref) + except: + pref = 'UTF-8' + while True: + yield pref + return yield_preferredencoding().next() + + +def htmlentity_transform(matchobj): + """Transforms an HTML entity to a Unicode character. + + This function receives a match object and is intended to be used with + the re.sub() function. + """ + entity = matchobj.group(1) + + # Known non-numeric HTML entity + if entity in htmlentitydefs.name2codepoint: + return unichr(htmlentitydefs.name2codepoint[entity]) + + # Unicode character + mobj = re.match(ur'(?u)#(x?\d+)', entity) + if mobj is not None: + numstr = mobj.group(1) + if numstr.startswith(u'x'): + base = 16 + numstr = u'0%s' % numstr + else: + base = 10 + return unichr(long(numstr, base)) + + # Unknown entity in name, return its literal representation + return (u'&%s;' % entity) + + +def sanitize_title(utitle): + """Sanitizes a video title so it could be used as part of a filename.""" + utitle = re.sub(ur'(?u)&(.+?);', htmlentity_transform, utitle) + return utitle.replace(unicode(os.sep), u'%') + + +def sanitize_open(filename, open_mode): + """Try to open the given filename, and slightly tweak it if this fails. + + Attempts to open the given filename. If this fails, it tries to change + the filename slightly, step by step, until it's either able to open it + or it fails and raises a final exception, like the standard open() + function. + + It returns the tuple (stream, definitive_file_name). + """ + try: + if filename == u'-': + if sys.platform == 'win32': + import msvcrt + msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) + return (sys.stdout, filename) + stream = open(encodeFilename(filename), open_mode) + return (stream, filename) + except (IOError, OSError), err: + # In case of error, try to remove win32 forbidden chars + filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename) + + # An exception here should be caught in the caller + stream = open(encodeFilename(filename), open_mode) + return (stream, filename) + + +def timeconvert(timestr): + """Convert RFC 2822 defined time string into system timestamp""" + timestamp = None + timetuple = email.utils.parsedate_tz(timestr) + if timetuple is not None: + timestamp = email.utils.mktime_tz(timetuple) + return timestamp + +def simplify_title(title): + expr = re.compile(ur'[^\w\d_\-]+', flags=re.UNICODE) + return expr.sub(u'_', title).strip(u'_') + +def orderedSet(iterable): + """ Remove all duplicates from the input iterable """ + res = [] + for el in iterable: + if el not in res: + res.append(el) + return res + +def unescapeHTML(s): + """ + @param s a string (of type unicode) + """ + assert type(s) == type(u'') + + htmlParser = HTMLParser.HTMLParser() + return htmlParser.unescape(s) + +def encodeFilename(s): + """ + @param s The name of the file (of type unicode) + """ + + assert type(s) == type(u'') + + if sys.platform == 'win32' and sys.getwindowsversion().major >= 5: + # Pass u'' directly to use Unicode APIs on Windows 2000 and up + # (Detecting Windows NT 4 is tricky because 'major >= 4' would + # match Windows 9x series as well. Besides, NT 4 is obsolete.) + return s + else: + return s.encode(sys.getfilesystemencoding(), 'ignore') + +class DownloadError(Exception): + """Download Error exception. + + This exception may be thrown by FileDownloader objects if they are not + configured to continue on errors. They will contain the appropriate + error message. + """ + pass + + +class SameFileError(Exception): + """Same File exception. + + This exception will be thrown by FileDownloader objects if they detect + multiple files would have to be downloaded to the same file on disk. + """ + pass + + +class PostProcessingError(Exception): + """Post Processing exception. + + This exception may be raised by PostProcessor's .run() method to + indicate an error in the postprocessing task. + """ + pass + +class MaxDownloadsReached(Exception): + """ --max-downloads limit has been reached. """ + pass + + +class UnavailableVideoError(Exception): + """Unavailable Format exception. + + This exception will be thrown when a video is requested + in a format that is not available for that video. + """ + pass + + +class ContentTooShortError(Exception): + """Content Too Short exception. + + This exception may be raised by FileDownloader objects when a file they + download is too small for what the server announced first, indicating + the connection was probably interrupted. + """ + # Both in bytes + downloaded = None + expected = None + + def __init__(self, downloaded, expected): + self.downloaded = downloaded + self.expected = expected + + +class YoutubeDLHandler(urllib2.HTTPHandler): + """Handler for HTTP requests and responses. + + This class, when installed with an OpenerDirector, automatically adds + the standard headers to every HTTP request and handles gzipped and + deflated responses from web servers. If compression is to be avoided in + a particular request, the original request in the program code only has + to include the HTTP header "Youtubedl-No-Compression", which will be + removed before making the real request. + + Part of this code was copied from: + + http://techknack.net/python-urllib2-handlers/ + + Andrew Rowls, the author of that code, agreed to release it to the + public domain. + """ + + @staticmethod + def deflate(data): + try: + return zlib.decompress(data, -zlib.MAX_WBITS) + except zlib.error: + return zlib.decompress(data) + + @staticmethod + def addinfourl_wrapper(stream, headers, url, code): + if hasattr(urllib2.addinfourl, 'getcode'): + return urllib2.addinfourl(stream, headers, url, code) + ret = urllib2.addinfourl(stream, headers, url) + ret.code = code + return ret + + def http_request(self, req): + for h in std_headers: + if h in req.headers: + del req.headers[h] + req.add_header(h, std_headers[h]) + if 'Youtubedl-no-compression' in req.headers: + if 'Accept-encoding' in req.headers: + del req.headers['Accept-encoding'] + del req.headers['Youtubedl-no-compression'] + return req + + def http_response(self, req, resp): + old_resp = resp + # gzip + if resp.headers.get('Content-encoding', '') == 'gzip': + gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r') + resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code) + resp.msg = old_resp.msg + # deflate + if resp.headers.get('Content-encoding', '') == 'deflate': + gz = StringIO.StringIO(self.deflate(resp.read())) + resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code) + resp.msg = old_resp.msg + return resp -- cgit v1.2.3 From 9beb5af82ecfbb35ee534692e73fb38f4399698a Mon Sep 17 00:00:00 2001 From: Filippo Valsorda Date: Fri, 13 Apr 2012 22:09:24 +0200 Subject: some HTMLParser bugfixes --- youtube_dl/utils.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'youtube_dl/utils.py') diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index a19656000..0f903c64a 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -73,7 +73,7 @@ def htmlentity_transform(matchobj): # Unknown entity in name, return its literal representation return (u'&%s;' % entity) - +HTMLParser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix class IDParser(HTMLParser.HTMLParser): """Modified HTMLParser that isolates a tag with the specified id""" def __init__(self, id): @@ -83,8 +83,17 @@ class IDParser(HTMLParser.HTMLParser): self.depth = {} self.html = None self.watch_startpos = False + self.error_count = 0 HTMLParser.HTMLParser.__init__(self) + def error(self, message): + print self.getpos() + if self.error_count > 10 or self.started: + raise HTMLParser.HTMLParseError(message, self.getpos()) + self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line + self.error_count += 1 + self.goahead(1) + def loads(self, html): self.html = html self.feed(html) -- cgit v1.2.3 From 921a14559277cfb2ead84b613c02d0251aeeb9ab Mon Sep 17 00:00:00 2001 From: Filippo Valsorda Date: Tue, 1 May 2012 17:01:51 +0200 Subject: dropped the support for Python 2.5 let's elaborate the decision: Python 2.5 is a 6 years old release and "under the current release policy, no security issues in Python 2.5 will be fixed anymore" (!!); also, it doesn't support the new zipfile distribution format. --- youtube_dl/utils.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'youtube_dl/utils.py') diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 0f903c64a..6e982157c 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -11,16 +11,12 @@ import sys import zlib import urllib2 import email.utils +import json try: import cStringIO as StringIO except ImportError: import StringIO - -try: - import json -except ImportError: # Python <2.6, use trivialjson (https://github.com/phihag/trivialjson): - import trivialjson as json std_headers = { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1', -- cgit v1.2.3 From 0b8c922da91fb7238ea15434d6a4792da84015bf Mon Sep 17 00:00:00 2001 From: Filippo Valsorda Date: Wed, 9 May 2012 09:41:34 +0000 Subject: Introduced Trouble(Exception) for more elegant non-fatal errors handling --- youtube_dl/utils.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'youtube_dl/utils.py') diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 6e982157c..d18073d72 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -290,6 +290,13 @@ class ContentTooShortError(Exception): self.expected = expected +class Trouble(Exception): + """Trouble helper exception + + This is an exception to be handled with + FileDownloader.trouble + """ + class YoutubeDLHandler(urllib2.HTTPHandler): """Handler for HTTP requests and responses. -- cgit v1.2.3 From 2c288bda4235bed6927d88d9bf53ecaec18f7904 Mon Sep 17 00:00:00 2001 From: Filippo Valsorda Date: Wed, 9 May 2012 14:47:28 +0200 Subject: reorganized the titles sanitizing: now title is the untouched title and stitle is created in process_info() and is cross-filesystem sanitized by sanitize_filename(); closes #164 --- youtube_dl/utils.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) (limited to 'youtube_dl/utils.py') diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index d18073d72..ae30da53e 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -156,12 +156,6 @@ def clean_html(html): return html -def sanitize_title(utitle): - """Sanitizes a video title so it could be used as part of a filename.""" - utitle = unescapeHTML(utitle) - return utitle.replace(unicode(os.sep), u'%') - - def sanitize_open(filename, open_mode): """Try to open the given filename, and slightly tweak it if this fails. @@ -196,10 +190,14 @@ def timeconvert(timestr): if timetuple is not None: timestamp = email.utils.mktime_tz(timetuple) return timestamp - -def simplify_title(title): - expr = re.compile(ur'[^\w\d_\-]+', flags=re.UNICODE) - return expr.sub(u'_', title).strip(u'_') + +def sanitize_filename(s): + """Sanitizes a string so it could be used as part of a filename.""" + def replace_insane(char): + if char in u' .\\/|?*<>:"' or ord(char) < 32: + return '_' + return char + return u''.join(map(replace_insane, s)).strip('_') def orderedSet(iterable): """ Remove all duplicates from the input iterable """ -- cgit v1.2.3