Getting Bookmark Data from del.icio.us

{
   "comment":"Docker Compose - Docker Documentation",
   "add_epoch":"1431953037",
   "title":"Docker Compose - Docker Documentation",
   "url":"http://docs.docker.com/compose/",
   "tags":[
      "docker",
      "development",
      "python"
   ],
   "icon_uri":"http://docs.docker.com/favicons/favicon-32x32.png",
   "private":0,
   "add_date":"2015-05-18 12:43:57",
 "icon_data":"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAFA0lEQVR4Ae3SA3wjXwLA8d/DzERNUiRre3u2bdu2bRsfnG3btm2st7x/ze2myejhdtvz5Wx8B89+/N///c8T/JUud5u7iPpXPqOimRlMrcbKpS7nAMdfSFzpmtflr+GCaEu6a/BZttrfK5O2CydOvxD4Kn8hbeoN/hqy06qY3g3Xs/XGRpHGTs9Pvpe/gnZRiT8lXJru1VlSBrAIvDPO66gpnJXgEd4B3kN35e9+OpDxqu12RLrnK+/jT1m8+m0fcXbjzjt7MOUsQVX7ECYPEmv6k7hNEK8S5RndRDOjxdUr3uxJMmn/EPi9wfTqlW7On5IXyv1Jub4D7wgLZUSxDN6RxR1MnuGEwOrA04WtNS7cufQNHiDSdqvni++YAL7Fb9CyfYauhNxiwsI+4ZxXebZJeocHpMnwogfhHdIaCAtredKaw3h/ZWlzp/L0x8AqgCtVt9rqQMXWB+vR6JF79X3tvVOmOiAqc2PzQEcPDH2fbqYvfLXrL+44/CIbhBSzJKwVSiAELmnT6rQQ1lBWiqhYRgghl3YefmweFh9VaC0tNIZ/fCPgpwAiT86KLM18VNYrOy5868nmrqu5sCDr00PHGqe+81S9tOtidOOkUjaMyudCbKFEEITgPecn5KxBeI8PIhRiLTRRKbI6wISFEiBgXTRy5FQ0+qPhVnPHhWd2XaxqvK9GcYvK0tSMyuIVfe7HL3kpg6TauJEt9myUcKUgz7EB6CxFxm0QAp0laBUgnUOlMSiNzBLC83U8hCYvmFrztj6IruiKlYH4ote4bGvrwZ2r85O+f3p4Wptktjo7+o3+kR+9zOlwSFzpejfGBpEEUWwMfUccucGDPrVykatfcaBYYp0A79cDBMI7cBZUAHh+SZgMr0NAAB4Q+LBArEPmckPzB5/+ycHPvu4uKksmzuUnQBtA2yCqAJcAf8fFnRf9vsrTiuqcJZg6icxTsu2DuEIZrEV2WoQTx5Gry6T7Lo2tDiCcRc+OEo4fIdt2CNPYjg9L65d0cZLi6E9pFHtoDH//2+2+zWWv9L2AHuBNwLi4/I1v1Q/sAMrKZHa5sftxkzsveWOKEUiJVwEoDXgwBpxBAEiF1wF4jzDZWgis5XmpIMkg6aDzlC0X/OjrtfmxJ9og6gAR4IFFYPT8BPhNcbn3/qcPX+vl7f7NQhYC8KzxQrBHpPRrz/fzAs57wLNO/HrnrcXFCRiDQLBh5uTEjqFv3An4Ml1opwJ+U+/C2Cf6FsaOxFHPhZ0rILRCWEtkUppByoBy6GAzuXf8mscbi8/ytQ/vkUBj7vTcttHvPNop/WXoTlz6lnfmdxnnbt+759ArN+/aXZNK0ChF9NeqvH6pyAVOYZ1jjXPrA+cGbwx4D0IS5jHN2dOnN4//8LFX+/DbPww46E48RvJ7Xiu1vvY973ffq1z72k89NHi4US6VUErxvqElXvSTGVLnEd7hnQMHCNYEJqN6drY1MHv6I82p488HfsqfIK58/ZvQzVc/+RF1lVve4eqXuuxlHn5ocPBKzQ3NalQs8tmftXjHqUVmE4P0FmVzorSdldtLk9UzU1+pL46/Y9PEsa8CMX8GcbNLXYQ/5ivLpjJ40YteYuv27Veq9/UdqteqtZNLsfrRXCsJ0vZKkCdDxc7ykcrK7E+3ThybAHL+AuLOB7fy53rriQkJqN0C7iDwOx0OcPDXE6/jX0vy/wn8fwL/Yj8H6qJUvBfQ9+kAAAAASUVORK5CYII="
}

import datetime
import urllib
import ssl
import json
import base64
import codecs
import logging
from urlparse import urlparse
from bs4 import BeautifulSoup
logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG)

url = 'https://del.icio.us/chrisramsay?&page=1'
try:
    r = urllib.urlopen(url)
except IOError:
    logging.warning('Could not open %s', url)
else:
    soup = BeautifulSoup(r, 'html.parser')
    bookmarks = soup.find_all("div", class_="articleThumbBlockOuter")
    print 'We have {} bookmarks'.format(len(bookmarks))

entry = {}
bookmark_zero = bookmarks[0]
title = bookmark_zero.find_all('div', class_='articleTitlePan')[0]
entry['title'] = title.a.attrs['title']
print entry

href = bookmark_zero.find_all('div', class_='articleInfoPan')[0].find_all('p')[0]
entry['url'] = href.a.attrs['href']
print entry

parsed = urlparse(entry['url'])
entry['icon_uri'] = u'{}://{}/favicon.ico'.format(parsed.scheme,  parsed.netloc)

entry['add_date'] = str(datetime.datetime.fromtimestamp(int(bookmark_zero.attrs['date'])))
entry['add_epoch'] = bookmark_zero.attrs['date']
print entry

try:
    tags = bookmark_zero.find_all('ul', class_='tagName')[0].find_all('li')
except IndexError:
    tags = []
entry['tags'] = [f.a.text for f in tags]
print entry

comment = None
for a in [l for l in bookmark_zero.find('div', class_='thumbTBriefTxt').children]:
    try:
        comment = a.p.contents[0]
    except AttributeError:
        continue
if comment is not None:
    entry['comment'] = comment
else:
    entry['comment'] = entry['title']
print entry

json.dumps(entry)

def get_icon(fetch):
    """Fetch an icon, base64 encode and return it."""
    icon_path = None
    icon_data = {'icon_uri': None, 'icon_data': None}
    # Fetch the root page, parse the HTML and get the LINK elements within HEAD.
    try:
        r_icon = urllib.urlopen(fetch)
    except (IOError, ssl.CertificateError):
        # Total fail, just return empty data
        return icon_data
    else:
        # Carry on & get the icon rel link
        icon_soup = BeautifulSoup(r_icon, 'html.parser')
        head = icon_soup.find('head')
        if head is None:
            return icon_data
        link = head.find('link', rel='icon')
        if link is not None:
            logging.debug('Icon URL: %s', link.attrs['href'])
            # Check for relative or absolute paths
            if ':' in link.attrs['href']:
                icon_path = link.attrs['href']
            else:
                icon_path = u'{}{}'.format(fetch, link.attrs['href'])
    # If we have icon_path, get the icon data
    if icon_path is not None:
        icon_data['icon_uri'] = icon_path
        try:
            icon_ulib = urllib.urlopen(icon_path)
        except (IOError, ssl.CertificateError):
            return icon_data
        # Get the content type to avoid things like 404 HTML
        content_type = icon_ulib.headers.getheader('Content-Type')
        if 'image' in icon_ulib.headers.getheader('Content-Type'):
            icon_data['icon_data'] = 'data:{};base64,{}'.format(
                content_type, base64.b64encode(icon_ulib.read()))
    # We are done
    return icon_data

def get_entry(bookmark):
    """Parses an individual del.icio.us entry"""
    entry = {}
    # Locate and set Title
    title = bookmark.find_all('div', class_='articleTitlePan')[0]
    entry['title'] = title.a.attrs['title']
    # Locate and set bookmark URL
    href = bookmark.find_all('div', class_='articleInfoPan')[0].find_all('p')[0]
    entry['url'] = href.a.attrs['href']
    logging.debug('Getting URL: %s', entry['url'])
    # Parse the URL - we need this to get the Icon
    parsed = urlparse(entry['url'])
    # Get the icon URL and base64 encoded icon image
    icon = get_icon(u'{}://{}'.format(parsed.scheme, parsed.netloc))
    entry['icon_uri'] = icon['icon_uri']
    entry['icon_data'] = icon['icon_data']
    # Get the date the bookmark was added, both in readable and epoch format
    entry['add_date'] = str(datetime.datetime.fromtimestamp(int(bookmark.attrs['date'])))
    entry['add_epoch'] = bookmark.attrs['date']
    # Just set the bookmark to public
    entry['private'] = 0
    # Get all the tags (if there are any)
    try:
        tags = bookmark.find_all('ul', class_='tagName')[0].find_all('li')
        entry['tags'] = [tag.a.text for tag in tags]
    except IndexError:
        entry['tags'] = ['none']
    # Get the comment (if there is one, else use the title)
    comment = None
    for brief_text in [l for l in bookmark.find('div', class_='thumbTBriefTxt').children]:
        try:
            comment = brief_text.p.contents[0]
        except AttributeError:
            continue
    if comment is not None:
        entry['comment'] = comment
    else:
        entry['comment'] = entry['title']
    # We are done
    return entry

link = json.dumps(get_entry(bookmarks[2]))
print link

filename = '/testing/links.json'

# Write starting character
with open(filename, 'w') as start:
    start.write('[')

# Here you might have something that fetched a page at a time and created
# a list of bookmarks per page - then interate through that list next.

# For each, blah, append links - add a comma after each bookmark except for the last bookmark
# Here's hoping that the last list item is not a failure.
for idx, bookmark in enumerate(bookmarks):
    with codecs.open(filename, 'a', encoding='utf-8') as incr:
        incr.write(json.dumps(get_entry(bookmark)))
        if idx < len(bookmarks) - 1:
            incr.write(',')

# Done, we write the ending character
with open(filename, 'a') as end:
    end.write(']')

<DT><A HREF="http://www.something.com/some-article/" ADD_DATE="1395356195" PRIVATE="0" TAGS="stuff,things" ICON_URI="http://www.something.com/favicon2.ico" ICON="data:image/x-icon;base64,[...]>Article Title</A>

bm_load = json.loads(link)
outlink = u'<DT><A HREF="{}" ADD_DATE="{}" PRIVATE="0" TAGS="{}" ICON_URI="{}" ICON="{}>{}</A>'.format(
    bm_load['url'], bm_load['add_epoch'], ','.join(bm_load['tags']), bm_load['icon_uri'],
    bm_load['icon_data'], bm_load['comment']
)
print outlink

import codecs
bookmarks = []
with codecs.open('/testing/links.json', 'r', encoding='utf8') as rfh:
    for bookmark in json.loads(rfh.read()):
        bookmarks.append(u'<DT><A HREF="{}" ADD_DATE="{}" PRIVATE="0" TAGS="{}" ICON_URI="{}" ICON="{}">{}</A>\n'.format(
            bookmark['url'], bookmark['add_epoch'], ','.join(bookmark['tags']), bookmark['icon_uri'],
            bookmark['icon_data'], bookmark['comment']
        ))

header = u"""
<!DOCTYPE NETSCAPE-Bookmark-file-1>
<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8">
<TITLE>Bookmarks</TITLE>
<H1>Bookmarks</H1>
<DL><p>
"""

with codecs.open('/testing/links.html', 'w', encoding='utf8') as wfh:
    wfh.write(header)
    for line in bookmarks:
        wfh.write(line)