Output errors to standard error

[software/subdivxget.git] / subdivxget

diff --git a/subdivxget b/subdivxget
index 5c5cc7a49cfab50003860880a380ff6737f67fdd..e938ac8e3b01c1d2254134922889f98573619c3b 100755 (executable)
--- a/subdivxget
+++ b/subdivxget
@@ -1,18 +1,42 @@
 #!/usr/bin/env python
 
 import sys
 #!/usr/bin/env python
 
 import sys

-import urllib
+if sys.version_info[0] < 3:
+       from HTMLParser import HTMLParser
+       from urllib import urlopen, urlretrieve, urlencode
+       def get_encoding(info):
+               return info.getparam('charset')
+
+else:
+       from html.parser import HTMLParser
+       from urllib.request import urlopen, urlretrieve
+       from urllib.parse import urlencode
+       def get_encoding(info):
+               return info.get_content_charset('ascii')

 import zipfile
 import subprocess
 import zipfile
 import subprocess

-import HTMLParser
+
+
+def output(fo, fmt, *args, **kargs):
+       if not args:
+               args = kargs
+       fo.write((fmt % args) + '\n')
+
+def echo(fmt, *args, **kargs):
+       output(sys.stdout, fmt, *args, **kargs)
+
+def error(fmt, *args, **kargs):
+       output(sys.stderr, fmt, *args, **kargs)
+

 
 class SubDivXQuery:
 
 class SubDivXQuery:

-       def __init__(self, to_search):
+       def __init__(self, to_search, page_number):

                self.host = "www.subdivx.com"
                self.page = "/index.php"
                self.down_page = "/bajar.php"
                self.query = dict(
                        buscar = to_search,
                self.host = "www.subdivx.com"
                self.page = "/index.php"
                self.down_page = "/bajar.php"
                self.query = dict(
                        buscar = to_search,

+                       pg = page_number,

                        accion = 5,
                        masdesc = '',
                        subtitulos = 1,
                        accion = 5,
                        masdesc = '',
                        subtitulos = 1,


@@ -21,117 +45,215 @@ class SubDivXQuery:
        @property
        def url(self):
                return 'http://%s%s?%s' % (self.host, self.page,
        @property
        def url(self):
                return 'http://%s%s?%s' % (self.host, self.page,

-                               urllib.urlencode(self.query))
+                               urlencode(self.query))

        @property
        def page_uri(self):
        @property
        def page_uri(self):

-               return self.page + '?' + urllib.urlencode(self.query)
+               return self.page + '?' + urlencode(self.query)

        @property
        def down_uri(self):
                return 'http://' + self.host + self.down_page
 
        @property
        def down_uri(self):
                return 'http://' + self.host + self.down_page
 

-class Subtitle:
-       pass

 
 

-class SubDivXHTMLParser(HTMLParser.HTMLParser):
+class SubDivXHTMLParser(HTMLParser):

 
        IDLE = 1
        HEADER = 2
 
        def __init__(self, down_uri):
 
        IDLE = 1
        HEADER = 2
 
        def __init__(self, down_uri):

-               HTMLParser.HTMLParser.__init__(self)
+               HTMLParser.__init__(self)

                self.down_uri = down_uri
                self.depth = 0
                self.parsing = False
                self.subs = []
                self.attr = None
                self.down_uri = down_uri
                self.depth = 0
                self.parsing = False
                self.subs = []
                self.attr = None

+               self.attr_depth = 0
+               self.cur = None
+               self.in_script_style = False

 
        def handle_starttag(self, tag, attrs):
                attrs = dict(attrs)
                if tag == 'div' and attrs.get('id') == 'menu_detalle_buscador':
 
        def handle_starttag(self, tag, attrs):
                attrs = dict(attrs)
                if tag == 'div' and attrs.get('id') == 'menu_detalle_buscador':

-                       #self.cur = Subtitle()

                        self.cur = dict()
                        self.subs.append(self.cur)
                        self.parsing = True
                if not self.parsing:
                        return
                        self.cur = dict()
                        self.subs.append(self.cur)
                        self.parsing = True
                if not self.parsing:
                        return

+               if tag == 'script' or tag == 'style':
+                       self.in_script_style = True
+                       return

                if tag == 'div':
                        if attrs.get('id') == 'buscador_detalle':
                                self.parsing = True
                        elif attrs.get('id') == 'buscador_detalle_sub':
                                self.attr = 'desc'
                if tag == 'div':
                        if attrs.get('id') == 'buscador_detalle':
                                self.parsing = True
                        elif attrs.get('id') == 'buscador_detalle_sub':
                                self.attr = 'desc'

+                               self.attr_depth = self.depth + 1
+                               self.cur[self.attr] = ''

                elif tag == 'a':
                        if attrs.get('class') == 'titulo_menu_izq':
                elif tag == 'a':
                        if attrs.get('class') == 'titulo_menu_izq':

-                               self.attr = 'title'
+                               self.attr = 'titulo'
+                               self.attr_depth = self.depth + 1
+                               self.cur[self.attr] = ''

                        elif attrs.get('href', '').startswith(self.down_uri):
                                self.cur['url'] = attrs['href']
                        elif attrs.get('href', '').startswith(self.down_uri):
                                self.cur['url'] = attrs['href']

-               if self.parsing:
+               # br are usually not closed, so ignore them in depth calculation
+               if self.parsing and tag != 'br':

                        self.depth += 1
 
        def handle_endtag(self, tag):
                if self.parsing:
                        self.depth += 1
 
        def handle_endtag(self, tag):
                if self.parsing:

-                       self.depth -= 1
+                       if tag == 'script' or tag == 'style':
+                               self.in_script_style = False
+                               return
+                       if self.depth == self.attr_depth:
+                               self.attr = None
+                               self.attr_depth = 0
+                       # see comment in handle_starttag()
+                       if tag != 'br':
+                               self.depth -= 1

                if self.depth == 0:
                        self.parsing = False
 
        def handle_data(self, data):
                if self.depth == 0:
                        self.parsing = False
 
        def handle_data(self, data):

-               if self.parsing:
-                       data = data.strip()
-                       if self.attr is not None and data:
-                               self.cur[self.attr] = data
+               if not self.parsing:
+                       return
+               data = data.strip()
+               # Hack to handle comments in <script> <style> which don't end
+               # up in handle_comment(), so we just ignore the whole tags
+               if self.in_script_style:
+                       return
+               if self.attr is not None and data:
+                       self.cur[self.attr] += ' ' + data
+                       if self.attr_depth == 0:
+                               self.cur[self.attr] = self.cur[self.attr].strip()

                                self.attr = None
                                self.attr = None

-                               #self.cur[self.attr] = self.cur.get(self.attr, '') + data.strip()
-                               #setattr(self.cur, self.attr, data.strip())
-                       elif data in ('Downloads:', 'Cds:', 'Comentarios:',
-                                       'Formato:'):
-                               self.attr = data[:-1].lower()
-                       elif data == 'Subido por:':
-                               self.attr = 'autor'
-                       elif data == 'el':
-                               self.attr = 'fecha'
+                               self.attr_depth = 0
+               elif data in ('Downloads:', 'Cds:', 'Comentarios:', 'Formato:'):
+                       self.attr = data[:-1].lower()
+                       self.attr_depth = 0
+                       self.cur[self.attr] = ''
+               elif data == 'Subido por:':
+                       self.attr = 'autor'
+                       self.attr_depth = 0
+                       self.cur[self.attr] = ''
+               elif data == 'el':
+                       self.attr = 'fecha'
+                       self.attr_depth = 0
+                       self.cur[self.attr] = ''

 
 
 
 

-def get_subs(query_str):
-       query = SubDivXQuery(query_str)
+def filter_subtitles(subs, filters):
+       def is_good(sub, filter):
+               def is_any_good(sub, filter):
+                       for value in sub.values():
+                               if value.lower().find(filter) >= 0:
+                                       return True

 
 

-       url = urllib.urlopen(query.url)
+               field = None
+               if len(filter) > 2 and filter[1] == ':':
+                       field = filter[0]
+                       filter = filter[2:]
+               filter = filter.lower()

 
 

-       parser = SubDivXHTMLParser(query.down_uri)
+               if field is None:
+                       return is_any_good(sub, filter)
+               elif field == 't':
+                       key = 'titulo'
+               elif field == 'd':
+                       key = 'desc'
+               elif field == 'a':
+                       key = 'autor'
+               elif field == 'f':
+                       key = 'formato'
+               elif field == 'c':
+                       key = 'comentarios'
+               elif field == 'C':
+                       key = 'cds'
+               elif field == 'F':
+                       key = 'fecha'
+               elif field == 'D':
+                       key = 'downloads'
+               else:
+                       # Not a recognizer field identifier, use the raw filter
+                       return is_any_good(sub, field + ':' + filter)
+
+               return sub[key].lower().find(filter) >= 0
+
+       if not filters:
+               return subs
+
+       result = []
+       for sub in subs:
+               for filter in filters:
+                       if not is_good(sub, filter):
+                               break
+               else:
+                       result.append(sub)
+       return result
+
+
+def subdivx_get_subs(query_str):
+       page_number = 1
+       subs = []
+       while True:
+               query = SubDivXQuery(query_str, page_number)
+               url = urlopen(query.url)
+               parser = SubDivXHTMLParser(query.down_uri)

 
 

-       for line in url:
-               parser.feed(line)
+               try:
+                       encoding = get_encoding(url.info())
+               except:
+                       encoding = 'ascii'

 
 

-       url.close()
+               for line in url:
+                       parser.feed(line.decode(encoding))

 
 

+               url.close()
+
+               if not parser.subs:
+                       break
+
+               subs.extend(parser.subs)
+               page_number += 1
+
+       return subs
+
+
+def get_subs(query_str, filters):
+       sub_exts = ('.srt', '.sub')

        zip_exts = ('application/zip',)
        rar_exts = ('application/rar', 'application/x-rar-compressed')
 
        zip_exts = ('application/zip',)
        rar_exts = ('application/rar', 'application/x-rar-compressed')
 

-       for sub in sorted(parser.subs, key=lambda s: int(s['downloads']), reverse=True):
-               print '''\
-       - %(title)s (%(autor)s - %(fecha)s - %(downloads)s - %(comentarios)s)
-         %(desc)s
-               DOWNLOADING ...
-       ''' % sub
-               fname, headers = urllib.urlretrieve(sub['url'])
+       subs = subdivx_get_subs(query_str)
+       subs = filter_subtitles(subs, filters)
+       subs.sort(key=lambda s: int(s['downloads']), reverse=True)
+
+       for sub in subs:
+               echo('''\
+- %(titulo)s (%(autor)s - %(fecha)s - %(downloads)s - %(comentarios)s)
+  %(desc)s
+       DOWNLOADING ...
+''', **sub)
+               fname, headers = urlretrieve(sub['url'])

                if 'Content-Type' in headers:
                        if headers['Content-Type'] in zip_exts:
                                z = zipfile.ZipFile(fname, 'r')
                                z.printdir()
                                for fn in z.namelist():
                if 'Content-Type' in headers:
                        if headers['Content-Type'] in zip_exts:
                                z = zipfile.ZipFile(fname, 'r')
                                z.printdir()
                                for fn in z.namelist():

-                                       if fn.endswith('.srt') or fn.endswith('.sub'):
+                                       if fn.endswith(sub_exts):

                                                if '..' in fn or fn.startswith('/'):
                                                if '..' in fn or fn.startswith('/'):

-                                                       print 'Dangerous file name:', fn
+                                                       error('Dangerous file name: %s', fn)

                                                        continue
                                                        continue

-                                               print 'Extracting', fn, '...'
+                                               echo('Extracting %s...', fn)

                                                z.extract(fn)
                        elif headers['Content-Type'] in rar_exts:
                                if subprocess.call(['rar', 'x', fname]) != 0:
                                                z.extract(fn)
                        elif headers['Content-Type'] in rar_exts:
                                if subprocess.call(['rar', 'x', fname]) != 0:

-                                       print 'Error unraring file %s' % fname
+                                       error('Error unraring file %s', fname)

                        else:
                        else:

-                               print 'Unrecognized file type:', headers['Content-Type']
+                               error('Unrecognized file type:',
+                                               headers['Content-Type'])

                else:
                else:

-                       print 'No Content-Type!'
+                       error('No Content-Type!')

 
 
 
 

-for q in sys.argv[1:]:
-       get_subs(q)
+get_subs(sys.argv[1], sys.argv[2:])