mklist-lovingtheclassics-pd

#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Fetch list of movies available from LovingTheClassics.  They claim
to have a list of DVDs with public domain movies.

Look up each title + year in IMDB and if only one title is returned
with the correct year, assume it is the correct IMDB title ID for
the movie.

"""

import argparse
import datetime
import json
import lxml.html
import movielib
import re
import urllib
import urllib2
import urlparse

from copy import deepcopy

def dumplist(list, name):
    outlist = deepcopy(list)
    # Rewrite freenessurl array to individual fields (freenessurl,
    # freenessurl2, freenessurl3...) in predictable/sorted order
    for i in outlist:
        if 1 == len(outlist[i]['freenessurl']):
            outlist[i]['freenessurl'] = outlist[i]['freenessurl'][0]
        else:
            sl = sorted(outlist[i]['freenessurl'])
            outlist[i]['freenessurl'] = sl[0]
            seq = 0
            for u in sl[1:]:
                field = "freenessurl%d" % (seq + 2)
                outlist[i][field] = u
                seq = seq + 1
    movielib.savelist(outlist, name=name)


def fetch_movie_list(args, l, url, name):
    try:
        root = lxml.html.fromstring(movielib.http_get_read(url))
    except urllib2.HTTPError as e:
        return None
    lastref = root.cssselect('p.amount strong')
    if lastref:
        lastinfo = lastref[0].text
        print("L: '%s'" % lastinfo)
        m = re.search("^(\d+) Item\(s\)$", lastinfo)
        last = int(m.group(1)) / 200 + 1
    else:
        last = 1
    page = 1
    while page <= last:
        for block in root.cssselect("li.item"):
            a = block.cssselect("h2.product-name a")[0]
            entryurl = urlparse.urljoin(url, a.attrib['href'])
            if is_in_list(l, entryurl):
                continue
            title = a.text_content().strip()
	    title = title.replace(' DVD-R', '')

            m = re.search("^(.+) \((\d{4})\)", title)
            if m:
                year = int(m.group(2))
                title = m.group(1)
            else:
                year = None

            imdb = entryurl
            entry = {
                'status': 'free',
                'freenessurl': [entryurl],
                'title': title,
                'updated': datetime.datetime.now().isoformat(),
            }
            if args.imdblookup:
                print("IMDB search for %s %s" % (title, year))
                searchtitle = title.replace(u"’", "'")\
                                   .replace(u"‘", "'")\
                                   .replace(u'“', '"')\
                                   .replace(u'”', '"')\
                                   .replace(u"–", "-")

                imdb = movielib.imdb_find_one(searchtitle, year)
                if imdb:
                    entry['imdblookup'] = '%s %d' % (searchtitle, year)
            if not imdb:
                imdb = entryurl
            if year:
                entry['year'] = year
            if imdb not in l:
                l[imdb] = entry
            else:
                if entryurl not in l[imdb]['freenessurl']:
                    l[imdb]['freenessurl'].append(entryurl)
                for f in ['updated', 'imdblookup']:
                    if f in entry:
                        l[imdb][f] = entry[f]
            print(imdb, l[imdb])
            dumplist(l, name=name)
        try:
            pageurl = url.replace('p=1', 'p=%d' % page)
            root = lxml.html.fromstring(movielib.http_get_read(pageurl))
        except urllib2.HTTPError as e:
            return None
        page += 1
    return l

def is_in_list(list, url):
    for e in list:
        if url in list[e]['freenessurl']:
            return True
    return False

def loadlist(l, path):
    try:
        with open(path, 'rt') as input:
            n = json.load(input)
            for id in n.keys():
                freenessurl = []
                for field in ['freenessurl', 'freenessurl2', 'freenessurl3',
                              'freenessurl4', 'freenessurl5', 'freenessurl6',
                              'freenessurl7', 'freenessurl8', 'freenessurl9']:
                    if field in n[id] and n[id][field] not in freenessurl:
                        freenessurl.append(n[id][field])
                        del n[id][field]
                n[id]['freenessurl'] = freenessurl
                if not id in l:
                    l[id] = n[id]
                else:
                    for url in n[id]['freenessurl']:
                        if url not in l[id]['freenessurl']:
                            l[id]['freenessurl'].append(url)
        return l
    except IOError as e:
        return l

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--imdblookup', action='store_true', default=False,
                        help='also find title IDs by searching for title/year in IMDB')
    args = parser.parse_args()
    url = "https://www.lovingtheclassics.com/pdmovies.html"
    url = "https://www.lovingtheclassics.com/pdmovies.html?limit=200&mode=grid&p=1"

    path='free-movies-loveingtheclassics-pd.json'
    outlist = {}
    #loadlist(outlist, path)
    outlist = fetch_movie_list(args, outlist, url, name=path)
    dumplist(outlist, name=path)

if __name__ == '__main__':
    main()