X-Git-Url: https://git.llucax.com/software/subdivxget.git/blobdiff_plain/f274666ba6db3400c33faf64aa40c3d2ba2d5156..1da4b46744b65f0e07b55dad1d6fdc882cce8a6d:/subdivxget diff --git a/subdivxget b/subdivxget index bd6fc6e..5d467ec 100755 --- a/subdivxget +++ b/subdivxget @@ -1,10 +1,20 @@ #!/usr/bin/env python import sys -import urllib +if sys.version_info[0] < 3: + from HTMLParser import HTMLParser + from urllib import urlopen, urlretrieve, urlencode + def get_encoding(info): + return info.getparam('charset') + +else: + from html.parser import HTMLParser + from urllib.request import urlopen, urlretrieve + from urllib.parse import urlencode + def get_encoding(info): + return info.get_content_charset('ascii') import zipfile import subprocess -import HTMLParser class SubDivXQuery: def __init__(self, to_search, page_number): @@ -22,27 +32,28 @@ class SubDivXQuery: @property def url(self): return 'http://%s%s?%s' % (self.host, self.page, - urllib.urlencode(self.query)) + urlencode(self.query)) @property def page_uri(self): - return self.page + '?' + urllib.urlencode(self.query) + return self.page + '?' + urlencode(self.query) @property def down_uri(self): return 'http://' + self.host + self.down_page -class SubDivXHTMLParser(HTMLParser.HTMLParser): +class SubDivXHTMLParser(HTMLParser): IDLE = 1 HEADER = 2 def __init__(self, down_uri): - HTMLParser.HTMLParser.__init__(self) + HTMLParser.__init__(self) self.down_uri = down_uri self.depth = 0 self.parsing = False self.subs = [] self.attr = None + self.attr_depth = 0 self.cur = None self.in_script_style = False @@ -62,12 +73,17 @@ class SubDivXHTMLParser(HTMLParser.HTMLParser): self.parsing = True elif attrs.get('id') == 'buscador_detalle_sub': self.attr = 'desc' + self.attr_depth = self.depth + 1 + self.cur[self.attr] = '' elif tag == 'a': if attrs.get('class') == 'titulo_menu_izq': - self.attr = 'title' + self.attr = 'titulo' + self.attr_depth = self.depth + 1 + self.cur[self.attr] = '' elif attrs.get('href', '').startswith(self.down_uri): self.cur['url'] = attrs['href'] - if self.parsing: + # br are usually not closed, so ignore them in depth calculation + if self.parsing and tag != 'br': self.depth += 1 def handle_endtag(self, tag): @@ -75,7 +91,12 @@ class SubDivXHTMLParser(HTMLParser.HTMLParser): if tag == 'script' or tag == 'style': self.in_script_style = False return - self.depth -= 1 + if self.depth == self.attr_depth: + self.attr = None + self.attr_depth = 0 + # see comment in handle_starttag() + if tag != 'br': + self.depth -= 1 if self.depth == 0: self.parsing = False @@ -88,15 +109,73 @@ class SubDivXHTMLParser(HTMLParser.HTMLParser): if self.in_script_style: return if self.attr is not None and data: - self.cur[self.attr] = data - self.attr = None - elif data in ('Downloads:', 'Cds:', 'Comentarios:', - 'Formato:'): + self.cur[self.attr] += ' ' + data + if self.attr_depth == 0: + self.cur[self.attr] = self.cur[self.attr].strip() + self.attr = None + self.attr_depth = 0 + elif data in ('Downloads:', 'Cds:', 'Comentarios:', 'Formato:'): self.attr = data[:-1].lower() + self.attr_depth = 0 + self.cur[self.attr] = '' elif data == 'Subido por:': self.attr = 'autor' + self.attr_depth = 0 + self.cur[self.attr] = '' elif data == 'el': self.attr = 'fecha' + self.attr_depth = 0 + self.cur[self.attr] = '' + + +def filter_subtitles(subs, filters): + def is_good(sub, filter): + def is_any_good(sub, filter): + for value in sub.values(): + if value.lower().find(filter) >= 0: + return True + + field = None + if len(filter) > 2 and filter[1] == ':': + field = filter[0] + filter = filter[2:] + filter = filter.lower() + + if field is None: + return is_any_good(sub, filter) + elif field == 't': + key = 'titulo' + elif field == 'd': + key = 'desc' + elif field == 'a': + key = 'autor' + elif field == 'f': + key = 'formato' + elif field == 'c': + key = 'comentarios' + elif field == 'C': + key = 'cds' + elif field == 'F': + key = 'fecha' + elif field == 'D': + key = 'downloads' + else: + # Not a recognizer field identifier, use the raw filter + return is_any_good(sub, field + ':' + filter) + + return sub[key].lower().find(filter) >= 0 + + if not filters: + return subs + + result = [] + for sub in subs: + for filter in filters: + if not is_good(sub, filter): + break + else: + result.append(sub) + return result def subdivx_get_subs(query_str): @@ -104,11 +183,16 @@ def subdivx_get_subs(query_str): subs = [] while True: query = SubDivXQuery(query_str, page_number) - url = urllib.urlopen(query.url) + url = urlopen(query.url) parser = SubDivXHTMLParser(query.down_uri) + try: + encoding = get_encoding(url.info()) + except: + encoding = 'ascii' + for line in url: - parser.feed(line) + parser.feed(line.decode(encoding)) url.close() @@ -118,40 +202,45 @@ def subdivx_get_subs(query_str): subs.extend(parser.subs) page_number += 1 - return sorted(subs, key=lambda s: int(s['downloads']), reverse=True) + return subs -def get_subs(query_str): +def get_subs(query_str, filters): + sub_exts = ('.srt', '.sub') zip_exts = ('application/zip',) rar_exts = ('application/rar', 'application/x-rar-compressed') - for sub in subdivx_get_subs(query_str): - print '''\ - - %(title)s (%(autor)s - %(fecha)s - %(downloads)s - %(comentarios)s) - %(desc)s - DOWNLOADING ... - ''' % sub - fname, headers = urllib.urlretrieve(sub['url']) + subs = subdivx_get_subs(query_str) + subs = filter_subtitles(subs, filters) + subs.sort(key=lambda s: int(s['downloads']), reverse=True) + + for sub in subs: + print('''\ +- %(titulo)s (%(autor)s - %(fecha)s - %(downloads)s - %(comentarios)s) + %(desc)s + DOWNLOADING ... +''' % sub) + continue + fname, headers = urlretrieve(sub['url']) if 'Content-Type' in headers: if headers['Content-Type'] in zip_exts: z = zipfile.ZipFile(fname, 'r') z.printdir() for fn in z.namelist(): - if fn.endswith('.srt') or fn.endswith('.sub'): + if fn.endswith(sub_exts): if '..' in fn or fn.startswith('/'): - print 'Dangerous file name:', fn + print('Dangerous file name:', fn) continue - print 'Extracting', fn, '...' + print('Extracting', fn, '...') z.extract(fn) elif headers['Content-Type'] in rar_exts: if subprocess.call(['rar', 'x', fname]) != 0: - print 'Error unraring file %s' % fname + print('Error unraring file %s' % fname) else: - print 'Unrecognized file type:', headers['Content-Type'] + print('Unrecognized file type:', headers['Content-Type']) else: - print 'No Content-Type!' + print('No Content-Type!') -for q in sys.argv[1:]: - get_subs(q) +get_subs(sys.argv[1], sys.argv[2:])