#!/cygdrive/c/python32/python
import pdb
from datetime import date, timedelta
import subprocess
import math

# Master program to make the various bird pages including:
# birdand place pages, bibliography, index, favorites, class pages,
# recent pictures, etc.  It does not do the tides.  There are two
# databases for the individual pictures (inventory.txt for birds and
# places.txt for places).  Both are tab delimited excel spreadsheet.
# The databases that defined how birds/places are grouped are called
# the classification databases and are bird_class and place_class.
# These databases are provide the breeding dates and which continent
# the birds are from.

# delete beginning and ending spaces and carriage return
def astrip(line):
    line = line.rstrip('\n')
    line = line.lstrip()
    line = line.rstrip()
    return line

# returns false if it is a comment line or blank
def isline(line):
    if len(line) == 0:
        return False
    elif line.startswith("#"):
        return False
    return line

# Add spaces to a 'string' that has multiple lines in it.  Spaces are
# added to the front of each line.
def add_space(line, space):
    new = '\n' + space
    aline = space + line        # add space to beginning
    aline = aline.rstrip('\n')  # temp. delete \n
    aline = aline.replace('\n', new)
    aline = aline + '\n'
    return aline

# read in the template html files used for the various pages
# returns a string
def get_html (filename):
    html = open(filename).read()
    return html

# Read in the info files for videos, feet and tongues.  Each line in
# these files has the uncompressed birdname, an item number(1, 2, etc.)
# and any text to be printed.  The information is saved in dictionaries.
# Since there can be multiple videos, etc., the dictionary key points to
# an array.  The item number with the compress bird/place name points to
# the actual text or picture file.
def read_text(dict, filename):
    dict_file = open(filename)
    for line in dict_file:
        line = astrip(line)
        aline = line.split('\t')
        bline = aline[1] + ":" + aline[2]
        if not aline[0] in dict:               # Initialize bird/place
            dict[aline[0]] = []                # in dictionary with array
        dict[aline[0]].append(bline)
    dict_file.close()

# Take the information for an individual picture and format it with text and
# highslide information.  All pics except classification pages use highslide.
# If I am doing 'Recent' or 'Favorites', the comments are replaced with the
# bird name.
def format_pic (line, num, flag):
                # true, if Favorites or Recents
    if flag:
        cap = line[4]
    else:
        if len(line) < 7:       # handles when there is no pict. comment
            cap = ""
        else:
            cap = line[6]
    num += 1
    year = line[2][:4]      # move date yyyymmdd to dd/mm/yyyy
    month = line[2][4:6]
    day = line[2][6:]
    date = month + '/' + day + '/' + year
    full = year + '/' + month + '/Images/' + line[3] + '.jpg'
    thumb = year + '/' + month + '/Images_sm/' + line[3] + '.jpg'
                # format the line
    aline = bird_pic.format(date=date,full=full,thumb=thumb,num=num,\
        place=line[5],caption=cap,bird=line[4])
    return aline

# Convert a string made up of multiple lines to a dictionary.  The string
# is a 'ls' of the files in the video, tongue and feet directories. The file
# names are of the form birdname_#.ext.  I want to convert the name portion
# to a dictionary key (a birdname without spaces) pointing to the full name,
# which is a list because I may have multiple videos, etc. of a specific bird.
# This dictionary is used to find out which videos, tongue and feet need to
# be printed in the bird pages.
def lstodic(ls_in):
    array = []
    dic = {}
    ls = ls_in.split('\n')
    for line in ls:
        beg = line.rfind('/')
        i = line.find('_')
        aline = line[beg+1:i]
        if not aline in dic:
            dic[aline] = []
        dic[aline].append(line)
    return dic
            
# Routine for reading in the bird and places databases. The lines
# are stored in dictionaries where the key is the uncompressed
# birdname or place.  From the bird database also store the info
# for the favorite birds (1 ranking) and recent birds (photo take
# less than 75 days ago).  I rearrange the data for sorting purposes.
#   Before:  date,priority,breeding,image,name,place,notes
#   After:   breeding,priority,date,image,name,place,notes
#
# The 'flag' variable keeps track of which database reading.
# If reading bird database, look for 'recents' and 'favorites' pics.

def read_data(dbase, filename, favorites, recents):
    flag = False
    if "inventory" in filename: flag = True     # inventory is birds

    dd = str(date.today()-timedelta(days=75))   # compute date 75 days ago
    dd = dd.replace("-", "")

    filein = open(filename)
    for line in filein:
        line = astrip(line)
        if not isline(line): continue           # get rid of comment and blank lines
        aline = line.split('\t')
        aline[2], aline[1], aline[0] = aline[0], aline[1], aline[2]
        if not aline[4] in dbase:               # new bird, initialize bird/place in
            dbase[aline[4]] = []                # dictionary with an array

        dbase[aline[4]].append(aline)           # add bird line to the dictionary
        if flag:
            if int(aline[1]) == 1:              # check for favorites
                favorites.append(aline)
                        # check for recents, the last one of the lowest priority
                        # will be the 'recent' picture.
            if int(aline[2]) > int(dd):
                if aline[4] in recents:
                    #pdb.set_trace()
                    if int(aline[1]) < int(recents[aline[4]][1]):
                        recents[aline[4]] = aline 
                else:
                    recents[aline[4]] = aline 
    filein.close()
    keys = list(dbase.keys())                   # sort the arrays in the dictionary
    for key in keys:
        dbase[key].sort()

# Formats the html for the right menu for birds only.  Input is one bird
# classification and formats the html into the right menu.  The right menu
# for birds is more complex because if the bird has multiple words in its name,
# last word comes first.
def fix_rightmenu_b (bird, menu):
    space14 = "              "
    space16 = "                "
    space18 = "                  "
    start = space14 + '<dl>\n'

    # t1 header for multiple birds in same species (multiple words) - no link
    # t2 birds with mutliple words & more than one in the species - used with t1
    # t3 bird with one word names & multiple name birds with only one in species
    t1 = space16 + '<dt>' + '{0}</dt>\n'
    t2 = space16 + '<dt>' + '&nbsp;&nbsp;&nbsp;&nbsp;<a class="right"\n' + \
            space18 + 'href="bird_{0}.shtml">{1}</a></dt>\n'
    t3 = space16 + '<dt>' + '<a class="right"\n' + space18 + \
            'href="bird_{0}.shtml">{1}</a></dt>\n'

    end = '              </dl>\n' + \
          '            </td>\n' + \
          '          </tr>\n' + \
          '        </table>\n' + \
          '      </td>\n'

    lines = menu + start
    keys = list(bird.keys())
    keys.sort()

    for k in keys:
            #pdb.set_trace()
            if not len(bird[k]):        # single word name
                line = t3.format(k, k)
                lines += line
            elif len(bird[k]) == 1:     # multiple words but only one family
                fname = bird[k][0]
                name = k + ', ' + fname
                linkn = fname +  k
                line = t3.format(linkn, name)
                lines += line
            else:                       # family name plus multiple birds
                line = t1.format(k)
                lines += line      
                alist=list(bird[k])
                alist.sort()
                for i in alist:
                    linkn = i.replace(' ', '')
                    linkn = linkn +  k
                    line = t2.format(linkn, i)
                    lines += line
    lines += end
    return lines           

# Formats the html for the right menu for places only.  Input is one place
# classification and formats the html into the right menu.  The right menu
# for places is simple because there is no last name first as with the birds.
# I also use <ul> it helps on formating the longer place names
def fix_rightmenu_p (bird, menu):
    space14 = "              "
    space16 = "                "
    space18 = "                  "
    start = space14 + '<ul>\n'
        # t1 head for multiple birds in same species (multiple words) - no link
        # t2 birds with mutliple words and more than one in the species - used with t1
        # t3 bird with one word names and multiple name birds with only one in species
    t3 = space16 + '<li>' + '<a class="right"\n' + space18 + \
            'href="bird_{0}.shtml">{1}</a></li>\n'

    end = '              </ul>\n' + \
          '            </td>\n' + \
          '          </tr>\n' + \
          '        </table>\n' + \
          '      </td>\n'

    lines = menu + start
    keys = list(bird.keys())
    keys.sort()

    for k in keys:
        link = k.replace(' ','')
        line = t3.format(link, k)
        lines += line

    lines += end
    return lines           


# Read in the classification databases (place_class, bird_class) which are 
# used for the right menu, the bird index, and providing the breeding dates for the
# bird web pages and creating the classification pages.
#
# On exit, the right_menu will be the dictionary that maps the individual
# bird/places to the appropriate classification.  For example, Common Loon
# is in 'Grebes and Loons'.  Each entry in right_menu will be formatted with
# the appropriate html. index will hold the information to create the index,
# class_dic connects each bird/place to the appropriate classification
# and the breeding status for appropriate birds is placed in breeding.
# In location is kept where the bird is found, currently North American or not.
#
# Also, save the bird names for the index.  bird_class also has alternate names
# and these are saved at this time for the index.  The bird names and alternates
# are stored in a dictionary where the element pointed to is the url for the bird
# page

def read_class (right_menu, class_in, index, class_dic, breeding, location):
            # load html for start of each right_menu
    menu = get_html("html_templates/right_menu.html")
            # rmenu hold the current classification that I am working on
    rmenu = ""

    for infile in ['bird_class', 'place_class']:
        flag = infile.replace('_class', '')   # keeps track if bird or place

            # read the bird and the place classification databases & process them
            # title is the name of the current classification being worked on and
            # is the text on the 'title:' lines.
        file_class = open(infile)

        title = ''
        for line in file_class:
            line = astrip(line)
            if not isline(line): continue           # get rid of comment and blank lines
            aline = line.split(':')

            if aline[0] == "title":
                if title:           # process the previous title information
                    if 'bird' in flag:
                        rmenu = fix_rightmenu_b(data,rmenu)
                    else:
                        rmenu = fix_rightmenu_p(data,rmenu)
                    right_menu[title] = rmenu
                title = astrip(aline[1])
                class_in[title] = []
                                # both clear out and intialize rmenu
                rmenu = menu.format(title)
                data = {}       # initialize and clear hash used to store data for right-menu
            elif aline[0] == "header":
                class_in[title].append(astrip(line))
            elif aline[0] == 'bird' or aline[0] == 'place':
                name = astrip(aline[1])
                class_in[title].append(astrip(line))
                        # Make sure I have the right number of fields for the birds
                        # Also, store where the birds are from (north america,
                        # etc.) and the breeding dates
                if flag == 'bird':
                                # if not enough fields, print out an error but continue
                                # by filling out remaining fields with blanks
                    if len(aline) < 6:
                        print ('following line does not have enough fields:' + line + '\n')
                        while (len(aline) < 6):
                            aline.append(' ')
                    breeding[name] = aline[4].replace('+', ':')
                                # check to make sure the continent is a digit
                    if not aline[3].isdigit() or int(aline[3]) > 6:
                        print ('error in continent, not digit' + line)
                        aline[3] = 0
                                # now count the bird based on the continent number
                    if not aline[3] in location:
                        location[aline[3]] = 0
                    location[aline[3]] += 1
                        
                        # link the bird or place to right classification
                class_dic[name] = title        

                        # now look for bird names with multiple words.
                        # If so, reverse so the last name is first. This is
                        # used in the right_menu and added to the index dictionary
                        # This is only done for birds.
                pdb.set_trace()
                if ' ' in name and flag == 'bird':
                    ind = name.rfind(" ")
                    lastname = name[ind+1:]
                    firstname = name[:ind]
                    if lastname not in data:
                        data[lastname] = []
                    data[lastname].append(firstname)
                    if flag == 'bird':          # only add birds to index
                        iname = lastname + ', ' + firstname
                        index{iname} = name
                else:
                    data[name] = []         # one word name has a zero length list
                    if flag == 'bird':      # only add birds to index
                        index{name} = name  # save names for index page

        if 'bird' in flag:
            rmenu = fix_rightmenu_b(data,rmenu)
        else:
            rmenu = fix_rightmenu_p(data,rmenu)
        right_menu[title] = rmenu

    keys = list(location.keys())
    keys.sort()
    for i in keys:
        if i == '1': print ('North American: ' + str(location[i]))
        if i == '2': print ('South American: ' + str(location[i]))
        if i == '3': print ('Europe:         ' + str(location[i]))
        if i == '4': print ('Africa:         ' + str(location[i]))
        if i == '5': print ('Asian:          ' + str(location[i]))
        if i == '6': print ('Australia:      ' + str(location[i]))
        if i == '0': print ('Not a bird (0): ' + str(location[i]))

# read in the bibliographic database and put it in a dictory stored by the
# refernce number which is the type (book, journal, web) + its number
def read_biblio(biblio, filename):
    biblio_file = open(filename)
    for line in biblio_file:
        line = astrip(line)
                        # replace ampersands with the html version &amp;
        line = line.replace('&', '&amp;')
        if not isline(line): continue
        aline = line.split('\t')
        if 'Books:' in aline[0]:
            type = 'b'
        elif 'Web:' in aline[0]:
            type = 'w'
        elif 'Journals:' in aline[0]:
            type = 'j'
        else:
            index = type + aline[0]
            if 'b' in type:
                if (len(aline) < 4):
                    print ("error in:", aline)
                    continue
                ref = aline[1] +  ', <i>' + aline[2] + '</i>,' + aline[3]
            elif 'w' in type:
                if (len(aline) < 3):
                    print ("error in:", aline)
                    continue
                ref = '<a href="' + aline[1] + '">' + aline[1] + '</a> ' +\
                        aline[2]
            elif 'j' in type:
                if (len(aline) < 5):
                    print ("error in:", aline)
                    continue
                ref = aline[1] + ', "' + aline[2] + '" <i>' + aline[3] + \
                        '</i>,' + aline[4]
            biblio[index] = ref
    biblio_file.close()

#   routine to print the bird pictures, sorted by their age (Juvenile and Adult
#   or breeding status.
def print_birds(outfile, bird_lines, birdname, b_dates):
    age = {'1':'Juvenile', '2':'Breeding', '3':'Non-breeding', '4':'Adult'}
    sp10 = '          '
    start = sp10 + '<div class="highslide-gallery">\n' + sp10 + '<table style="width:435px;">\n'

    title = sp10 + '<tr class="extras">\n' + sp10 + \
              '<th colspan="3"><b class="largef">{title}</b></th>\n' + sp10 + \
            '</tr>\n'
            # need to figure out if there are breeding dates and if so, put them
            # in the age field. If there are no breeding dates, dates will have the
            # length of 1.  Need to pad that out.
    dates = b_dates.split(':')
    if len(dates) < 2: dates.append('')

            # Now add the breeding dates (or blanks if none) to the
            # breeding, non-breeding titles
    if dates[0]: age['2'] = age['2'] + ' (' + dates[0] + ')'
    if dates[1]: age['3'] = age['3'] + ' (' + dates[1] + ')'

    keys = list(bird_lines.keys())
    keys.sort()
    for key in keys:
        #print (birdname + ' = ' + key)
        k = int(key)
        if k > 0 and k < 5:
            header = title.format (title=age[key])
        else:
            header = title.format(title=' ')
        outfile.write(start)
        outfile.write(header)
        blist = bird_lines[key]
        num = len(blist)
        ind = 0
        while ind < num:
            outfile.write(sp10 + '<tr>\n')
            if ind < num:
                outfile.write(blist[ind])
            if ind+1 < num:
                outfile.write(blist[ind+1])
            if ind+2 < num:
                outfile.write(blist[ind+2])
            outfile.write(sp10 + '</tr>\n')
            ind += 3
    outfile.write (sp10 + '</table>\n')
    outfile.write (sp10 + '</div>\n')
 
# print out pics, either 3 or 4 columns per row.  Used for
# places (3), favorites (4), recents (4), etc.
def print_pics (n_pic, outfile, pic_lines, start, blank_pic):
    sp10 = '          '
    #pdb.set_trace()
    num = len(pic_lines)
            # make the input (pic_lines) a multiple of 4
            # to ease column problem in html.  This is mainly
            # true when num of pic_lines is less than 4
    while num % n_pic:
        pic_lines.append(' ')
        num += 1

    outfile.write(start)
    num = len(pic_lines)
    i = 1
    for line in pic_lines:
        if i % n_pic == 1:
            outfile.write(sp10 + '<tr>\n')
        if line == " ":
            outfile.write(blank_pic)
        else:
            outfile.write(line)
        if i % n_pic == 0:
            outfile.write(sp10 + '</tr>\n')
        i += 1
    outfile.write (sp10 + '</table>\n')
    if 'div' in start: outfile.write (sp10 + '</div>\n')
        

# Print out the second and final part of a complete description.
# The main thing to do is the bibliography and the surrounding
# html text.
def process_end(lines, biblio, abird):
    space10 = '          '
    space8  = '        '
    info1 = space8 + '<a name="moreinfo"></a>\n'
    info2 = space8 + '<div class="small">\n' + space10 + \
            '<table style="width:435px;">\n'
    info3 = space10 + '<tr>\n' + space10 + '<td>\n'
    pstart= space10 + '<p>\n'
    pend  = space10 + '</td>\n' + space10 + '</tr>\n' + space10 + \
            '</table>\n' + space10 + '</div>\n'
    pboth = space10 + '</p>\n' + space10 + '<p>\n'

    out = []

    out.append(info1)
    out.append(info2)
    out.append(info3)
    out.append(pstart)

    while (lines):
        i = lines.pop(0)
        if 'Bibliography:' in i: break
        out.append (space10 + i)

            # now do the references, need to make sure there is no
            # extra spaces, the (b, j, w) are lower case, and it is
            # the key is is in the master bibliography database
            # Also, if it is a web bibliography look for the date
            # accessed

            # add header stuff for bibliography
    #pdb.set_trace()
    out.append(space10 + i)
    out.append(space10 + '<ul class="NoBullet">\n')

    while (lines):
                # always get the first line left
        access_date = ' '
        line1 = lines.pop(0)
        line1 = line1.rstrip('\n')
        line1 = line1.lstrip()
        line2 = line1.split(';')
        i = line2[0]
        if len(line2) > 1: access_date = ', Accessed ' + line2[1]
        i = i.rstrip()
        i = i.lower()
        #pdb.set_trace()
        if 'w' in i or 'b' in i or 'j' in i:
            if (i in biblio):
                out.append(space10 + '<li>' + biblio[i] + access_date + '</li>\n')
            else:
                print ('erroneous key: ' + i + ' for:' + abird)
        else:
            print ('erroneous key: ' + i + ' for:' + abird)

    out.append (space10 + '</ul>\n')
    out.append(pend)
    return out

# routine for checking out if a video, feet, or tongue are available
def find_extra (bird, abird, blist, tx_list, btype):
    text = {}
            # first see if there is any text for that bird and put
            # it into a dictionary based on the extension ('1', '2')
            # otherwise, create a text line to go with the video
    if tx_list:
        for line in tx_list:
            fields = line.split(':')
            text[fields[0]] = fields[1]
        if btype == 'videos': btype='film'
    else:
        if btype == 'videos':
            tx = 'Movie of ' + bird
            btype = 'film'
        elif btype == 'tongue':
            tx = 'Picture of ' + bird + "'s tongue"
        elif btype == 'feet':
            tx = 'Picture of ' + bird + "'s feet"

            # cycle through the items found in the 'ls'
    aline = ""
    for f in blist:
                    # get extension number
        i = f.find('_')
        ind = f[i+1]
                    # get rid of the leading '../'
        link = f[3:]
        if ind in text:
            tx_line = text[ind]
        else:
            tx_line = tx
        line = space10 + '<p><a class="nounder" href="' + link + '">'+ \
            '<img src="Icons/' + btype + '.png" alt="feet" />\n' + \
            space10 + tx_line + '</a></p>\n'
        aline = aline + line
    return aline

#
# main routine
#
birds = {}          # dictionary for birds
places = {}         # dictionary for places 
favorites = []      # favorite birds to process
recents = {}        # recent bird photos
right_menu={}       # right menu (all birds or places in a class)
index={}            # dictionary holding the bird index information
biblio={}           # bibliography database
breeding={}         # breeding/non-breeding dates where appropriate
location={}         # contains where the bird is found
tx_tongue={}        # storing text information for tongues
tx_feet={}          # storing text information for feets
dummy = []          # place holder for places
class_in = {}       # holds the classification and bird/places in it for printing
class_dic = {}      # maps a bird/place to the appropriate classification
                    # which then used to acces the correct right menu.
tx_video={}         # storing videos information
space10 = '          '

# read in all the databases
read_class (right_menu, class_in, index, class_dic, breeding, location)
read_data (birds, "inventory.txt", favorites, recents)
read_data (places, "place.txt", dummy, dummy)
read_text (tx_video, "videos.txt")
read_text (tx_tongue, "tongues.txt")
read_text (tx_feet, "feet.txt")
read_biblio (biblio, "bibliography.txt")

# get a list of videos, feet, tongue, Descriptions, and short text for birds
# Text is optional for the tongue and feet.  Also, complete description are
# gradually replacing short text (dir: text).
ls_text = str(subprocess.check_output(['ls', '../text/*.txt']), encoding='utf8')
ls_desc = str(subprocess.check_output(['ls', '../Descriptions/*.txt']), encoding='utf8')
ls_videos = str(subprocess.check_output(['ls', '../videos/*.mp4']), encoding='utf8')
ls_feet = str(subprocess.check_output(['ls', '../feet/*.jpg']), encoding='utf8')
ls_tongue = str(subprocess.check_output(['ls', '../Tongue/*.jpg']), encoding='utf8')
ls_place = str(subprocess.check_output(['ls', '../text_place/*.txt']), encoding='utf8')
ls_class = str(subprocess.check_output(['ls', '../text_class/*.txt']), encoding='utf8')
ls_title = str(subprocess.check_output(['ls', '../text_title/*.txt']), encoding='utf8')

# covert the lists in videos, feet & tongue to a dictionary
dic_videos = lstodic(ls_videos)
dic_feet = lstodic(ls_feet)
dic_tongue = lstodic(ls_tongue)

# read in the various html lines used to create the pages
basic_head = get_html("html_templates/basic_head.html")
basic_end = get_html("html_templates/basic_end.html")
head_bird = get_html("html_templates/bird_head.html")
end_bird =  get_html("html_templates/bird_end.html")
other_end = get_html("html_templates/other_end.html")
bird_pic = get_html("html_templates/pic.html")
ind_line = get_html("html_templates/bird_index.html")
ind_head = get_html("html_templates/index_head.html")
ind_end = get_html("html_templates/index_end.html")
class_head = get_html("html_templates/class_head.html")
class_end = get_html("html_templates/class_end.html")
class_pic = get_html("html_templates/class_pic.html")
blank_pic = get_html("html_templates/blank_pic.html")

video_line = get_html("html_templates/video_line.html")
video_txt = get_html("html_templates/video_text.html")
biblio_txt = get_html("html_templates/biblio_text.html")

# now start creating the web pages in this order, birds, places,
# videos, favorites, latest, bibliography and class pages
keys = list(birds.keys())
for bird in keys:
    atext=[]
    dtext=[]
    ahead = head_bird.format(bird=bird, menu='birds_menu.html')
                # bird name without any spaces
    abird = bird.replace(' ', '')
    filename = 'bird_' + abird + '.shtml'
    outfile = open(filename, 'w')
    outfile.writelines(ahead)
                # first print out the various info on each bird
    if abird in ls_desc:
                # a full description is divided into two sections, the
                # first is printed before the pictures, the second after
                # the pictures.  Up until the ':--' or Diet: is the 
                # first part.  If there is a complete description skip
                # the short one in ls_text
        filename = '../Descriptions/' + abird + '.txt'
        file_d = open(filename)
        outfile.write(space10 + '<div class="small" style="width:450px;">\n')
                # first print out the first part of the description
        for aline in file_d:
            if ':---' in aline: break
            if isline(astrip(aline)): outfile.write(space10 + aline)
                # add the internal link to the first part of the description.
        outfile.write(space10 + '<p><a href="#moreinfo">More info</a></p>\n')
        outfile.write(space10 + '</div>')

                # save the second part for later printing
        for aline in file_d:
            if isline(astrip(aline)):
                if (aline[0] == ':'):
                    aline = aline.rstrip('\n')
                    aline = aline[1:]
                    aline = '<b>' + aline + '</b>\n'
                atext.append(aline)
        dtext = process_end (atext, biblio, abird)
        file_d.close()

    elif abird in ls_text:      # short description
        outfile.write(space10 + '<div class="small" style="width:450px;">\n')
        file_d = open('../text/' + abird + '.txt')
        for line in file_d:
            outfile.writelines(space10 + line)
        outfile.write(space10 + '</div>\n')
        file_d.close()
    else:
        outfile.write(space10 + '<div class="small" style="width:450px;">\n')
        outfile.write(space10 + '<p>Pictures of have taken of ' + \
                abird + '.</p>\n')
        outfile.write(space10 + '</div>\n')
    outfile.write(space10 + '<div class="small" style="width:450px">\n')
    if abird in dic_videos:     # are there any videos
        if (abird in tx_video):
            out = find_extra (bird, abird, dic_videos[abird], tx_video[bird], 'videos')
        else:
            out = find_extra (bird, abird, dic_videos[abird], "", 'videos')
        outfile.writelines(out)
    if abird in dic_feet:       # are there any pictures of feet
        if (bird in tx_feet):
            out = find_extra (bird, abird, dic_feet[abird], tx_feet[bird], 'feet')
        else:
            out = find_extra (bird, abird, dic_feet[abird], "", 'feet')
        outfile.writelines(out)
    if abird in dic_tongue:     # are there any pictures of tongues
        if (bird in tx_tongue):
                out = find_extra (bird, abird, dic_tongue[abird], tx_tongue[bird], 'tongue')
        else:
            out = find_extra (bird, abird, dic_tongue[abird], "", 'tongue')
        outfile.writelines(out)
    outfile.write(space10 + '</div>\n')
                # not do the pictures, first sort each picture into its
                # proper category, juvenile, breeding, non-breeding or adult
                # not all categories will have pictures
    blist = birds[bird]
    blist.sort()
    num = 0
                # cycle through the bird pictures, formatting them and putting
                # them in bird_lines depending on their age (Juvenile, adult, etc)
    bird_lines = {}
    for line in blist:
                # call routine to format pic with all the needed html
        aline = format_pic(line, num, 0)
        num += 1
        if line[0] in bird_lines:
            bird_lines[line[0]].append(aline)
        else:
            bird_lines[line[0]] = []
            bird_lines[line[0]].append(aline)

                # find out if there are breeding dates
                # now print the bird images
    if bird in breeding:
        print_birds(outfile, bird_lines, bird, breeding[bird])
    else:
        print ('error in breeding key for : ' + bird)
        print_birds(outfile, bird_lines, bird, " ")
    if dtext:
                # print out the rest of the complete description
        outfile.write(space10 + '<div class="small" style="width:450px;">\n')
        outfile.writelines(dtext)
        outfile.write(space10 + '</div>')

    menu = right_menu[class_dic[bird]]
    outfile.writelines(menu)
    outfile.writelines(end_bird)
    outfile.close()

            # now put out the place pages, will cycle through the places
            # dictionary, printing one page at a time
keys = list(places.keys())
for place in keys:
    ahead = head_bird.format(bird=place, menu='places_menu.html')
            # place name without any spaces
    aplace = place.replace(' ', '')
    filename = 'bird_' + aplace + '.shtml'
    outfile = open(filename, 'w')
    outfile.writelines(ahead)
                # first print out the various info on each place
    outfile.write(space10 + '<div class="small" style="width:450px;">\n')
    if aplace in ls_place:
        file_d = open('../text_place/' + aplace + '.txt')
        for line in file_d:
            outfile.writelines(space10 + line)
        file_d.close()
    else:
        outfile.writelines(space10 + '<p>This page contains pictures I have taken of' \
                + place + '<\p>\n')
    outfile.writelines(space10 + '</div>')
            # now format the pictures, putting them into an array before printing
    place_lines = []
    num = 0
    for line in places[place]:
        aline = format_pic (line, num, 0)
        num += 1
        place_lines.append(aline)

    start = '''
        <div class="highslide-gallery">
          <table style="width:435px;">'''
    #pdb.set_trace()
    print_pics (3, outfile, place_lines, start, blank_pic) 
    menu = right_menu[class_dic[place]]
    outfile.writelines(menu)
    outfile.writelines(end_bird)
    outfile.close()

            # now do the favorites page
favorites.sort(reverse=True)
ahead = head_bird.format(bird='Favorites', menu='birds_menu.html')
filename = 'bird_' + 'Favorites' + '.shtml'
outfile = open(filename, 'w')
outfile.writelines(ahead)
outfile.writelines (space10 + '<p>Below are images of birds that I especially liked </p>\n')
#outfile.writelines(space10 + '</div>')

fav = []
num = 0
for line in favorites:
    aline = format_pic (line, num, 1)
    num += 1
    fav.append(aline)
start = '''
        <div class="highslide-gallery">
          <table style="width:435px;">'''
print_pics (4, outfile, fav, start, blank_pic) 
outfile.write(space10 + '</div>\n')
outfile.writelines(other_end)

outfile.close()

            # now do the recent page
recl = list(recents.values())
recl.sort()
ahead = head_bird.format(bird='Recent Photos', menu='birds_menu.html')
filename = 'bird_' + 'Latest' + '.shtml'
outfile = open(filename, 'w')
outfile.writelines(ahead)
outfile.writelines (space10 + '<p>Below are some of the pictures of birds that I have' \
        ' recently taken.</p>\n')

rec = []
num = 0
count = 0
for line in recl:
    if count > 30: break
    aline = format_pic (line, num, 1)
    num += 1
    count += 1
    rec.append(aline)
start = '''
        <div class="highslide-gallery">
          <table style="width:435px;">'''
print_pics (4, outfile, rec, start, blank_pic)
outfile.write(space10 + '</div>\n')
outfile.writelines(other_end)

outfile.close()

        # now do the bibliography page
ahead = basic_head.format(title='Bibliography for Bird Journal')
filename = 'Bibliography' + '.shtml'
outfile = open(filename, 'w')
outfile.writelines(ahead)
outfile.writelines (biblio_txt)

bib_lines = {}
keys = biblio.keys()
btext = {'b':'<b> Books: </b>\n', 'j':'<b> Journals, Magazines </b>\n', 'w':'<b> Web: </b>\n'}
for k in keys:
    ind = k[0]
    if not ind in bib_lines:
        bib_lines[ind] = []
    bib_lines[ind].append(biblio[k])

alist = ['b', 'j', 'w']
for i in alist:
    outfile.write(space10 + '<img src="Icons/bird.png" alt="bird">' + \
            btext[i])
    outfile.write(space10 + '<ul class="NoBullet">\n')
    blist = []
    blist = bib_lines[i]
    blist.sort()
    for line in blist:
        outfile.write(space10 + '<li>' + line + '</li>\n')
    outfile.write(space10 + '</ul>\n')
    outfile.write(space10 + '</div>\n')

outfile.writelines(basic_end)
outfile.close()

        # now do the video page
ahead = basic_head.format(title="Bird Videos")

filename = 'videos' + '.shtml'
outfile = open(filename, 'w')
outfile.writelines(ahead)
outfile.writelines(video_txt)
outfile.write(space10 + '<table class="tvideo">\n')
keys = list(tx_video.keys())
keys.sort()
for k in keys:
    bname = k.replace(' ', '')
    for i in tx_video[k]:
        fields = i.split(':')
        lname = 'bird_' + bname + '.shtml'
        vname = 'videos/' + bname + '_' + fields[0] + '.mp4'
        line = video_line.format(link=lname, bird=k, video=vname, text= fields[1])
        outfile.write(line)
outfile.write(space10 + '</table>\n')
outfile.write(space10 + '</div>\n')
outfile.writelines(basic_end)
outfile.close()

        # now do the bird index page
pdb.set_trace()
space4 = '    '
space6 = '      '
filename = 'class_index.shtml'
outfile = open(filename, 'w')
outfile.writelines(ind_head)

        # put the index into a hash array depending on the first
        # letter of the last name
line1 = space4 + '<a href="#{letter}"><b style="color:magenta;">{letter}</b></a>&nbsp;&nbsp;\n'
line2 = space4 + \
        '<td style="vertical-align:top;"><a name="{letter}">' + \
        '<b style="color:magenta;">{letter}</b></a></td>\n' + space4 + '</tr>\n'
line3 = space6 + \
        '<td style="vertical-align:top;">\n' + \
        space6 + \
        '<a class="index" href="{link}">{bird}</a> </td>\n'
        # first sort the bird names into a dictionary based on the first letter
        # of the lastname.
alpha = {}
for ind in index:
    if not ind[0] in alpha:
        alpha[ind[0]] = []
    alpha[ind[0]].append(ind)

keys = list(alpha.keys())
keys.sort()
        # put out the internal jump index (e.g., 'A', 'B', etc.)
for i in keys:
    linef = line1.format(letter=i)
    outfile.write(linef)

        # now go through each letter of the alphabet and print the index
        # for that letter.
outfile.write(space4 + '<table>\n' + space4 + '<tr>\n')
for i in keys:
    linef = line2.format(letter=i)
    outfile.write(linef)
    lines = alpha[i]
    lines.sort()
    num = len(lines)                # total number of birds names to print
    rows = math.ceil(num/4)         # number of rows needed
    cols = 4
    #pdb.set_trace()
    outfile.write (space4 + '<tr>\n')
    for i1 in range(rows):
        for j1 in range(cols):
            ind = i1 + (rows * j1)
            if ind < num:
                fields = lines[ind].split(',')   # need to have first name first for link
                if len(fields) > 1:
                    link = 'bird_' + fields[1] + fields[0] + '.shtml'
                    link = link.replace(' ','')
                else:
                    link = 'bird_' + fields[0] + '.shtml'
                    link = link.replace(' ','')
                linef = line3.format(link=link, bird=lines[ind])
                outfile.write (linef)
        outfile.write (space6 + '</tr>\n')
outfile.writelines(ind_end)
outfile.close()

    # Lastly, print the classification pages
    # each classification page has one title (defnes the classification) and option
    # header which group the species
keys = list(class_in.keys())
keys.sort()
for key in keys:
    if 'place:' in class_in[key][1]:
            lmenu = 'places'
    else:
            lmenu = 'birds'
    pics = []
    akey = key.replace(' ', '')
    akey = akey.replace(',', '')
    ahead = class_head.format(aname=key, lmenu=lmenu)
    filename = 'class_' + akey + '.shtml'
    outfile = open(filename, 'w')
    outfile.write(ahead)
            # check to see if there is text for this classification
    if akey in ls_title:
                    # replace the <p> with a class to decrease right margin
        lines = get_html('../text_title/' + akey + '.txt')
        lines = lines.replace('<p>', '<p class="ptext">')
        lines = add_space(lines, '      ')
        outfile.write(lines)
            # now look for sub-classes and print the pics in these subclasses.
            # If there is no sub-class the text in the header line is blank.
    start = '    <table style="width:435px;">'
    class_pics = []
    flag = 0
    for line in class_in[key]:
        if 'header' in line:
            if (flag):
                print_pics (4, outfile, class_pics, start, blank_pic)
                class_pics = []
            aline = line.replace('header:','')
            if not isline(aline): continue
            bline = aline.replace(' ', '')
            bline = bline.replace(',', '')
            flag = 1
            outfile.write(space4 + '<h3>' + astrip(aline) + '</h3>\n')
                    # check for text for class
            temp = 'text_class/' + bline + '.txt'
            if temp in ls_class:
                #pdb.set_trace()
                lines = get_html ('../text_class/' + bline + '.txt')
                    # replace the <p> with a class to decrease right margin
                lines = lines.replace('<p>', '<p class="ptext">')
                lines = add_space(lines, '      ')
                outfile.write(lines)
        if 'bird:' in line or 'place:' in line:
            fields = line.split(':')
            link = 'bird_' + fields[1].replace(' ', '') + '.shtml'
            link = link.replace(',','')
            bline = class_pic.format(link=link, pic=fields[2], name=fields[1])
            class_pics.append(bline)
    print_pics (4, outfile, class_pics, start, blank_pic)
    outfile.write(class_end)

    outfile.close()