#!/cygdrive/c/python34/python
import pdb
from datetime import date, timedelta
from bird_utils import get_image_size
import subprocess
import math
import re

# Master program to make the various bird pages including:
# birdand place pages, bibliography, index, favorites, class pages,
# recent pictures, etc.  It does not do the tides.  There are two
# databases for the individual pictures (inventory.txt for birds and
# places.txt for places).  Both are tab delimited excel spreadsheet.
# The databases that defined how birds/places are grouped are called
# the classification databases and are bird_class and place_class.
# These databases are provide the breeding dates and which continents
# the birds are from.

# routine to get rid of spaces, commas and periods from a classification
# so I can use it as a filename
def class_strip(inv):
    prev = inv.replace(' ', '')
    prev = prev.replace(',', '')
    prev = prev.replace('.', '')
    return prev

# delete beginning and ending spaces and carriage return
def astrip(line):
    line = line.rstrip('\n')
    line = line.lstrip()
    line = line.rstrip()
    return line

# returns false if it is a comment line or blank
def isline(line):
    if len(line) == 0:
        return False
    elif line.startswith("#"):
        return False
    return line

# Add spaces to a 'string' that has multiple lines in it.  Spaces are
# added to the front of each line.
def add_space(line, space):
    new = '\n' + space
    aline = space + line        # add space to beginning
    aline = aline.rstrip('\n')  # temp. delete \n
    aline = aline.replace('\n', new)
    aline = aline + '\n'
    return aline

# read in the template html files used for the various pages
# returns a string
def get_html (filename):
    html = open(filename).read()
    return html

# Read in the info files for videos, feet and tongues.  Each line in
# these files has the uncompressed birdname, an item number(1, 2, etc.)
# and any text to be printed.  The information is saved in dictionaries.
# Since there can be multiple videos, etc., the dictionary key points to
# an array.  The item number with the compress bird/place name points to
# the actual text or picture file.
def read_text(dict, filename):
    dict_file = open(filename)
    for line in dict_file:
        line = astrip(line)
        aline = line.split('\t')
        bline = aline[1] + ":" + aline[2]
        if not aline[0] in dict:               # Initialize bird/place
            dict[aline[0]] = []                # in dictionary with array
        dict[aline[0]].append(bline)
    dict_file.close()

# Take the information for an individual picture and format it with text and
# highslide information.  All pics except classification pages use highslide.
# If I am doing 'Recent' or 'Favorites', the comments are replaced with the
# bird name.
# If I am putting out individual bird pictures, the maximum pixel width is 140.
# Otherwise, it is the saved width, which is 180 pixels.
def format_pic (line, num, num_pixels, flag):
                # true, if Favorites or Recents
    if flag:
        cap = line[4]
    else:
        if len(line) < 7:       # handles when there is no pict. comment
            cap = ""
        else:
            cap = line[6]
    num += 1
    year = line[2][:4]      # move date yyyymmdd to dd/mm/yyyy
    month = line[2][4:6]
    day = line[2][6:]
    date = month + '/' + day + '/' + year
    full = year + '/' + month + '/Images/' + line[3] + '.jpg'
    thumb = year + '/' + month + '/Images_sm/' + line[3] + '.jpg'
    		# get the width & height from num_pixels
    abird = year + month + ":" + line[3]
    if abird in num_pixels:
        stats = num_pixels[abird].split(":")
        width = stats[0]
        height = stats[1]
    else:
        print (" didn't find height and width for: ", line)
                # format the line
    aline = bird_pic.format(width=width,height=height,date=date,full=full,thumb=thumb,num=num,\
        place=line[5],caption=cap,bird=line[4])
    return aline

# Convert a string made up of multiple lines to a dictionary.  The string
# is a 'ls' of the files in the video, tongue and feet directories. The file
# names are of the form birdname_#.ext.  I want to convert the name portion
# to a dictionary key (a birdname without spaces) pointing to the full name,
# which is a list because I may have multiple videos, etc. of a specific bird.
# This dictionary is used to find out which videos, tongue and feet need to
# be printed in the bird pages.
def lstodic(ls_in):
    array = []
    dic = {}
    ls = ls_in.split('\n')
    for line in ls:
        beg = line.rfind('/')
        i = line.find('_')
        aline = line[beg+1:i]
        if not aline in dic:
            dic[aline] = []
        dic[aline].append(line)
    return dic
            
# Routine for reading in the bird and places databases. The lines
# are stored in dictionaries where the key is the uncompressed
# birdname or place.  From the bird database also store the info
# for the favorite birds (1 ranking) and recent birds (photo take
# less than 75 days ago).  I rearrange the data for sorting purposes.
#   Before:  date,priority,breeding,image,name,place,notes
#   After:   breeding,priority,date,image,name,place,notes
#
# The 'flag' variable keeps track of which database reading.
# If reading bird database, look for 'recents' and 'favorites' pics.
# Also, get the pixel dimensions for the small version of each pic and
# store by 'date:name'.

import openpyxl
from openpyxl import load_workbook
import numbers
def read_data(dbase, filename, sheet_nm, favorites, recents, num_pixels):
    flag = False
    if "inventory" in filename: flag = True     # inventory is birds

    dd = str(date.today()-timedelta(days=75))   # compute date 75 days ago
    dd = dd.replace("-", "")

    wb = openpyxl.load_workbook(filename, read_only=True)
    active = wb.active
    sheet = wb[sheet_nm] # ws is now an Iterableworksheet
    for row in sheet.rows:
        aline=[]
        num = 0
        for cell in row:
            acell = cell.value
            if acell == None:
                acell = ""          # if nothing cell, make it an empty string
            if isinstance(acell, numbers.Integral):
                aline.append(str(acell))
            else:
                aline.append(acell)
            num += 1
            if num > 7: break
        if  '#' in aline[0]: continue           # get rid of comment and blank lines
        if  len(aline) < 1:  continue           # get rid of comment and blank lines

    #pdb.set_trace()
        datef = astrip(aline[0])			# used a little later for getting pathname for file
        aline[2], aline[1], aline[0] = aline[0], aline[1], aline[2]
        if not aline[4] in dbase:               # new bird, initialize bird/place in
            dbase[aline[4]] = []                # dictionary with an array

        dbase[aline[4]].append(aline)           # add bird line to the dictionary
        if flag:
            if int(aline[1]) == 1:              # check for favorites
                favorites.append(aline)
                        # check for recents, the last one of the lowest priority
                        # will be the 'recent' picture.
            if int(aline[2]) > int(dd):
                if aline[4] in recents:
                    #pdb.set_trace()
                    if int(aline[1]) < int(recents[aline[4]][1]):
                        recents[aline[4]] = aline 
                else:
                    recents[aline[4]] = aline 
                # get the height, width of each image and put into a dictionary by date:picname
        if 'inventory' in filename or 'place' in filename:
             #pdb.set_trace()
             d_year = datef[0:4]
             d_month = datef[4:6]
             pic_index = d_year + d_month + ':' + aline[3]
             p_path = '../' + d_year + '/' + d_month + '/Images_sm/' + aline[3] + '.jpg'
             width, height = get_image_size(p_path)
             if not p_path in num_pixels:
                 num_pixels[pic_index] = str(width) + ':' + str(height)
             else:
                 print ('Problem with duplicate pathname for a file for pixel width, height: ', p_path)

    		# print out the database of picture pixels
#    outf = open ('picture_sizes', 'w')
#    for keys, values in num_pixels.items():
#        outf.write(keys+" "+values+"\n")
#    outf.close()
    keys = list(dbase.keys())                   # sort the arrays in the dictionary
    for key in keys:
        dbase[key].sort()

# Formats the html for the right menu for birds only.  Input is one bird
# classification and formats the html into the right menu.  The right menu
# for birds is more complex because if the bird has multiple words in its name,
# last word comes first.
def fix_rightmenu_b (bird, menu):
    space = " "
    #start = space*18 + '<ul id="rightmenu">\n'
    start = space*18 + '<ul>\n'


    # t1 header for multiple birds in same species (multiple words) - no link
    # t2 birds with mutliple words & more than one in the species - used with t1
    # t3 bird with one word names & multiple name birds with only one in species
    t1 = space*16 + '<li>' + '{0}\n'
    #t2 = space*16 + '<li>' + '<a class="right"\n' + \
    t2 = space*16 + '<li>' + \
            space*18 + '<a href="bird_{0}.shtml">&nbsp;&nbsp;&nbsp;{1}</a></li>\n'
    t3 = space*16 + '<li>\n ' +  space*18 + \
            '<a href="bird_{0}.shtml">{1}</a></li>\n'

    end = '              </ul>\n' + \
          '            </div>\n' + \
          '          </aside>\n' + \
          '          </section> <!-- end of main content and sidebar-->\n'

    lines = menu + start
    keys = list(bird.keys())
    keys.sort()

    for k in keys:
            if not len(bird[k]):        # single word name
                line = t3.format(k, k)
                lines += line
            elif len(bird[k]) == 1:     # multiple words but only one family
                fname = bird[k][0]
                name = k + ', ' + fname
                linkn = fname +  k
                linkn = linkn.replace(' ','')
                line = t3.format(linkn, name)
                lines += line
            else:                       # family name plus multiple birds
                line = t1.format(k)
                lines += line      
                lines += start
                alist=list(bird[k])
                alist.sort()
                for i in alist:
                    linkn = i.replace(' ', '')
                    linkn = linkn +  k
                    line = t2.format(linkn, i)
                    lines += line
                lines += space*16 + '</ul>'
                lines += space*16 + '</li>'
    lines += end
    return lines           

# Formats the html for the right menu for places only.  Input is one place
# classification and formats the html into the right menu.  The right menu
# for places is simple because there is no last name first as with the birds.
# I also use <ul> it helps on formating the longer place names
def fix_rightmenu_p (bird, menu):
    space = " "
    start = space*14 + '<ul>\n'
        # t3 bird with one word names and multiple name birds with only one in species
    t3 = space*16 + '<li>' + '<a class="right"\n' + space*18 + \
            'href="bird_{0}.shtml">{1}</a></li>\n'

    end = '              </ul>\n' + \
          '      </div>\n' \
          '      </aside>\n' \
          '      </section>\n'

    lines = menu + start
    keys = list(bird.keys())
    keys.sort()

    for k in keys:
        link = k.replace(' ','')
        line = t3.format(link, k)
        lines += line

    lines += end
    return lines           


def read_place_class(right_menu, class_in, class_dic):
# Read in the place classification/family database (place_class).  Besides
# providing the organization for the 'class_*.shtml' pages, I use this database
# to create the right menu for the individual 'place' pages. right_menu, class_in and
# class_dic store both the 'place' and 'bird' information
#
#   right_menu:  Holds all the places (and birds) in a 'family/class'.
#                Used for producing right_menu.  All the html is embedded.
#   class_in:    Has the title, header and 'bird/places' from the class databases
#                which is used for producing the family/class pages.
#   class_dic    Used to connect a specific bird/place to the appropriate right
#                menu which is organized by family/class.

            # load html for start of each right_menu
    menu = get_html("html_templates/right_menu.html")
            # rmenu hold the current classification that I am working on
    rmenu = ""
            # read and process the place classification database.
    file_class = open('place_class')
            # title is the name of the current classification being worked on and
            # is from the 'title:' line in the database.
    title = ''
    for line in file_class:
        line = astrip(line)
        if not isline(line): continue           # get rid of comment and blank lines
        aline = line.split(':')

        if aline[0] == "title":
            if title:           # if true, process the previous title information
                rmenu = fix_rightmenu_p(data,rmenu)
                right_menu[title] = rmenu
            title = astrip(aline[1])
            class_in[title] = []
                                # both clear out and intialize rmenu
            tlink = title.replace(' ', '')
            tlink = tlink.replace(',', '')
            tlink = tlink.replace('.', '')
            tlink = 'class_' + tlink + '.shtml'
                                # need to make sure the current family name is not too
                                # long.  If so, truncate and add an etc.
            margin1 = "-5"
            if len(title) > 25: margin1 = "-20"
            if len(title) > 30: margin1 = "-40"
            atitle = shorten(title)

            rmenu = menu.format(tlink,atitle,margin1)
            data = {}       # initialize and clear hash used to store data for right-menu
        elif aline[0] == "header":
            class_in[title].append(astrip(line))
        elif aline[0] == 'bird' or aline[0] == 'place':
            name = astrip(aline[1])
            class_in[title].append(astrip(line))
                        # link the place to right classification
            class_dic[name] = title        
            data[name] = []
    rmenu = fix_rightmenu_p(data,rmenu)
    right_menu[title] = rmenu

def cm2inch(num):
# convert from cm to feet/inches
#
# input:  can be of the form 'xxx' or 'xxx-xxx'
# returned: inches or feet along with the original cm

    line = ["", ""]             # holds the converted number(s)
    cline = ["", ""]            # holds the cm version passed in
    num = num.replace(' ', '')
    num = num.replace('cm','')
    fields = num.split('-')     # split up the possible mult. num.
    
    i = 0
    flag = 0
    for f in fields:
                    # convert string to a number
        line[i] = ""
        total = float(f)/2.54
                    # will convert to feet if the 1st number > 12 inches
        if total > 12 and i == 0:
            total = total/12
            flag = 1
        elif total > 12 and flag:
            total = total/12
        line[i] = '{0:.1f}'.format(total)
        cline[i] = '{0:.1f}'.format(float(f))
        i += 1
    if flag:
        type = ' ft'
    else:
        type = ' in'
    if line[1]:     # if multiple entries, put them together
        if float(line[1]) < float(line[0]):
            print ('in cm2inch: 2nd number < 1st: ', num)
        newline = '-'.join(line) + type + ' (' + '-'.join(cline) + ' cm)'
    else:
        newline = line[0] + type + ' (' + cline[0] + ' cm)'
    return newline

def ft2(num):
# Make sure feet is printed correctly, '23-45 ft' or 5 ft and get the
# metric equivalents
    line = ["", ""]
    cline = ["", ""]

    num = num.replace(' ', '')
    fields = num.split("-")         # split up the multiple nums, if any
    i = 0
    for f in fields:
        line[i] = ""
        if 'in' in f:               # handle the case xft23in
            subf = f.split("ft")
            feetp = subf[0]
            inch = subf[1].replace('in','')
            total = (float(feetp) * 12 + int(inch))/12.
        else:
            feetp = f.replace("ft", "")
            total = float(feetp)
        line[i] = '{0:.1f}'.format(total)
        cline[i] = '{0:.1f}'.format(total*30.48)
        i += 1
    if line[1]:
        newline = '-'.join(line)
        cnewline = '-'.join(cline)
        newline = newline + ' ft' + ' (' + cnewline + ' cm)'
                    # quick check to make sure the 2nd number larger than 1st
        if float(line[1]) < float(line[0]):
            print ('For feet, the second less than 1st: ', num)
    else:
        newline = line[0] + ' ft (' + cline[0] + ' cm)'
    return newline

def g2lb(num):
#   convert from grams to ounces
    line = ["", ""]
    lbflag = 0
    num = num.replace(' ', '')
    factor = 1
    if 'kg' in num: factor = 1000
    num = num.replace('kg', '')
    num = num.replace('g', '')
    fields = num.split("-")         # split up the multiple nums, if any
    i = 0
    for f in fields:
        #pdb.set_trace()
        line[i] = ""
        oz = float(f)*factor*.035274
        if oz > 16 and (i < 1 or lbflag):   # convert to pds if over 16 oz
            lbflag = 1                      # and the minimum is also over
            oz = oz/16                      # 16 oz
        line[i] = '{0:.1f}'.format(oz)
        i += 1
    if line[1]:
        newline = '-'.join(line)
    else:
        newline = line[0]
    if lbflag:
        newline = newline + ' lb' + ' (' + num + ' g)'
    else:
        newline = newline + ' oz' + ' (' + num + ' g)'
    return newline

def lb2(num):
#   convert from lbs OR ounces to grams
    line = ["", ""]
    cline = ["", ""]

    num = num.replace(' ', '')
    fields = num.split("-")         # split up the multiple nums, if any
    i = 0
    for f in fields:
        line[i] = ""
        if 'oz' in num:               # handle the case xft23in
            grams = f.replace('oz','')
            grams = float(grams)*28.3495
        elif 'lb' in num:
            grams = f.replace("lb", "")
            grams = float(grams)*16*28.3495
        line[i] = '{0:.1f}'.format(grams)
        i += 1
    newline = line[0]
    if line[1]:
        newline = '-'.join(line)
    newline = ' (' + newline + ' g)'

    return newline
def in2in (num):
#   while no conversion to inches, need to handle multiple values
    line = ["", ""]
    cm = ["", ""]
    num = num.replace(' ', '')
    num = num.replace('in', '')
    fields = num.split("-")         # split up the multiple nums, if any
    i = 0
    flag = 0
    type = ' in'
    for f in fields:
        #pdb.set_trace()
        line[i] = ""
        value = float(f)
        cm[i] = '{0:.1f}'.format(value*2.54)
        if i == 0 and value > 12:
            flag = 1
            value = value/12
            type = ' ft'
        elif i == 1 and flag:
            value = value/12
            type = ' ft'
        line[i] = '{0:.1f}'.format(value)
        i += 1
    if line[1]:                     # multiple values, join them
        newline = '-'.join(line) + type
        cmline = ' (' + '-'.join(cm) + ' cm)'
    else:
        newline = line[0] + type
        cmline = ' (' + cm[0] + ' cm)'
    newline = newline + cmline
    return newline



# read in the database containing various facts and breeding dates
# for each bird. If the value exists, only then will the dictionary
# be populated for that bird
# Read/outputed:
# blength:      length of each bird
# bwingspan:    wingspan of each bird
# bweight:      weight of each bird
# beggs:        clutch size for each bird
# bstatus:      conservation status of each bird
# breeding      breeding status for the birds


def read_bird_facts (blength, bwingspan, bweight, beggs, bchicks, bstatus, breeding):

    wbf = openpyxl.load_workbook('bird_facts.xlsx', read_only=True)
    activef = wbf.active
    sheetf = wbf['bird_facts'] # ws is now an Iterableworksheet

    for line in sheetf.rows:
        fields=[]
        anum = 0
        for cell in line:
            if cell.value == None:
                fields.append("")           # if nothing in cell, make it an empty string
            else:
                fields.append(cell.value)
        anum += 1
        #print (fields[0])
        if anum > 9:
            break

        name = astrip(fields[0])
                # process lengths and wingspan, converting metric to standard
        lengths = ['', '']
        for i in range(1,3):    # handle the possible two lengths
            num = fields[i].replace(' ','')
            if "N/A" in num:
                lengths[i-1] = num
            elif "ft" in num: 
                lengths[i-1] = ft2(num)
            elif 'cm' in num:
                lengths[i-1] = cm2inch(num)
            elif 'in' in num:
                lengths[i-1] = in2in(num)
            elif num:
                print (' error in lengths for: ', name)
        if lengths[1]:           # if true, have values for male and female
            blength[name] = 'M: ' + lengths[0] + ', F: ' + lengths[1]
        else:
            blength[name] = lengths[0]

                # now process wingspan
        num = astrip(fields[3])
        if num:
            if "N/A" in num:
                lengths[i-1] = num
            elif "ft" in num: 
                bwingspan[name] = ft2(num)
            elif 'cm' in num:
                bwingspan[name] = cm2inch(num)
            elif 'in' in num:
                bwingspan[name] = in2in(num)
            elif num:
                print (' error in wingspan for: ', name)

                # now do weight(s)
        weights = ['', '']
        j=0
        for i in range(4,6):    # handle two weights
            num = fields[i].replace(' ','')
            if "N/A" in num:
                weights[j] = num
            elif "NoInfo" in num:
                weights[j] = fields[i]
            elif "g" in num: 
                weights[j] = g2lb(num)
            elif 'lb' in num:
                num = num.replace('lb', ' lb')
                weights[j] = num + lb2(num)
            elif 'oz' in num:
                num = num.replace('oz', ' oz')
                weights[j] = num + lb2(num)
            elif num:
                print (' error in weights for: ', name)
            j+= 1
        if weights[1]:           # if true, have values for male and female
            bweight[name] = 'M: ' + weights[0] + ', F: ' + weights[1]
        elif fields[4] or fields[5]:
            bweight[name] = weights[0]
                # now do clutch size, chicks, conservation status and breeding dates
        eggs = fields[6].replace("--", "-")
        if astrip(fields[6]): beggs[name] = astrip(eggs)
        if astrip(fields[7]): bchicks[name] = astrip(fields[7])
        if astrip(fields[8]): bstatus[name] = astrip(fields[8])
        if astrip(fields[9]): breeding[name] = fields[9].replace('+', ':')
        

# Read in the bird_class database which is used for the right_menu,
# the bird index, the 'Family All' page and providing the breeding dates
# for the bird web pages and creating the classification pages.
#
# On exit, the right_menu will be the dictionary that maps the individual
# bird/places to the appropriate classification.  For example, Common Loon
# is in 'Grebes and Loons'.  Each entry in right_menu will be formatted with
# the appropriate html. index will hold the information to create the index,
# class_dic connects each bird/place to the appropriate classification.
# 'continent' contains the continents found for each bird and cont_count, the
# number of birds found in each continent.
#
# Also, save the bird names for the index.  bird_class also has alternate names
# and these are saved at this time for the index.  The bird names and alternates
# are stored in a dictionary where the element pointed to is the url for the bird
# page.  'scien' contains the scientific name pointing to each bird and scien_name has the
# bird name pointing to the scientific name. The first is used for the scientific index
# and the second to put the scientific name on each bird page.
#
# Format of bird class:  type (bird, title, etc.), English name, Latin Name,
#                                image thumbnail, continent, Alternate names,
#                                (multiple names separated by '+')
#
# right_menu, class_in and class_dic store both the 'place' and 'bird' information
#
#   right_menu:  Holds all the places (and birds) in a 'family/class'.
#                Used for producing right_menu.  All the html is embedded.
#   class_in:    Has the title, header and 'bird/places' from the class databases
#                which is used for producing the family/class pages.
#   index        Stores the information for the English index
#   class_dic    Used to connect a specific bird/place to the appropriate right
#                menu which is organized by family/class.
#   alltitles    List of the all the bird families/titles.  This is used for
#                Previous/Next Family on the bird Family/class pages.
#   continent    Stores information on which continent(s) are recorded for each bird
#		 (list, '+' separated) - 0 (N/A), 1(NA),2(SA), 3(EU), 4(AF), 5(AS), 6(AU),
#                7(ANT), 8(oceanic), 9 (Island)
#   cont_count	 Number of birds found in each continent.
#   scien        Dictionary with scientific name pointing to english name
#   scien_name   Dictionary with birdname pointing to the scientific name
#
# Internal variables

def read_bird_class (right_menu, class_in, index, class_dic, alltitles, continent, cont_count,scien, scien_name):
    birdsnum = 0    # Count of the number of birds, minus non-birds (0 in continent),
                    # For example, Just for Fun pictures.
            # load html for start of each right_menu
    menu = get_html("html_templates/right_menu.html")
            # rmenu hold the current classification that I am working on
    rmenu = ""

            # read and process the bird classification database.
    file_class = open('bird_class')
            # title is the name of the current classification being worked on and
            # is from the 'title:' line in the database.
    title = ''
    #pdb.set_trace()
    for line in file_class:
        line = astrip(line)
        if not isline(line): continue           # get rid of comment and blank lines
        aline = line.split(':')
                                # Save the title and header lines for Family all page
#       if 'bird' in flag and 'title' in aline[0]:
#               families.append = '<h4>' + aline[1] + '</h4>'
#       if 'bird' in flag and 'head' in aline[0]:
#               families.append = '<a href=>' + aline[1] + '</h4>'

        if aline[0] == "title":
                                # save titles for later use in next and previous class pages
            t = astrip(aline[1])
            alltitles.append(t)
            if title:           # if true, process the previous title information
                rmenu = fix_rightmenu_b(data,rmenu)
                right_menu[title] = rmenu
            title = astrip(aline[1])
            class_in[title] = []
                                # both clear out and intialize rmenu
            tlink = title.replace(' ', '')
            tlink = tlink.replace(',', '')
            tlink = tlink.replace('.', '')
            tlink = 'class_' + tlink + '.shtml'
                                # need to make sure the current family name is not too
                                # long.  If so, truncate and add an etc.
            marginl = "-5"
            if len(title) > 25: margin1 = "-20"
            if len(title) > 30: margin1 = "-40"
            atitle = shorten(title)

            rmenu = menu.format(tlink,atitle,margin1)
            data = {}       # initialize and clear hash used to store data for right-menu
        elif aline[0] == "header":
            class_in[title].append(astrip(line))
        elif aline[0] == 'bird':
                        # first count the number of birds
            if not ('Just for' in aline[1] or 'Oil Spill' in aline[1]):
                birdsnum += 1
            name = astrip(aline[1])
                        # Make sure I have the right number of fields for the birds
                        # Also, store where the birds are from (north america,
                        # etc.) and the breeding dates
                        # if not enough fields, print out an error but continue
                        # by filling out remaining fields with blanks
            if len(aline) < 6:
                print ('following line does not have enough fields:' + line + '\n')
                while (len(aline) < 6):
                    aline.append(' ')
                        # store the line for later processing minus latin name
                        # and continents.  Not needed later and line better matches
                        # 'place' line so I can process both with the same code
            #pdb.set_trace()
            tmp = ':'.join([aline[0],aline[1],aline[3], aline[4]])
            class_in[title].append(tmp)
                                # Save the scientific name for each bird
            if not 'Oil Spill' in name and not 'Just for' in name:
                scien[astrip(aline[2])] = name
                scien_name[name] = astrip(aline[2])
                                # Now process the continent(s)
            continent[name] = aline[4]
            cfields = aline[4].split('+')
            for cc in cfields:
                                # make sure that the continent value is from 0-9
                if not cc.isdigit() or int(cc) > 9:
                    print ('error in continent, not digit' + line)
                    cc = 0
                else:
                    cc = int(cc)
                cont_count[cc] += 1
                        
                        # link the bird to right classification
            class_dic[name] = title        

                        # now look for bird names with multiple words.
                        # If so, reverse so the last name is first. This is
                        # used in the right_menu and added to the index dictionary
                        # This is only done for birds.
            if ' ' in name:
                ind = name.rfind(" ")
                lastname = name[ind+1:]
                firstname = name[:ind]
                if lastname not in data:
                    data[lastname] = []
                data[lastname].append(firstname)
                        # add bird to index
                iname = lastname + ', ' + firstname
                        # check to make sure I don't have duplicate names
                if not iname in index:
                    index[iname] = name # save name for index page
                else:
                     print ("duplicate name in bird_class: ", iname)
            else:
                data[name] = []         # one word name has a zero length list
                if not name in index:
                    index[name] = name      # save names for index page
                else:
                     print ("duplicate name in bird_class: ", name)

                        # now see if there are alternate names.  The keys for alternate
                        # names have and ending ':1', so they are sorted correctly later
                        # and I can recognize them and make them another color
            aline[5] = astrip(aline[5])
            if len(aline[5]) > 0:
                dline = aline[5].split('+')
                #pdb.set_trace()
                for bname in dline:
                    bname = astrip(bname)
                    if ' ' in bname:
                        ind = bname.rfind(" ")
                        lastname = bname[ind+1:]
                        firstname = bname[:ind]
                        iname = lastname + ', ' + firstname + ':1'
                        index[iname] = name
                    else:
                        index[bname+':1'] = name  # save names for index page

    rmenu = fix_rightmenu_b(data,rmenu)
    right_menu[title] = rmenu
    file_class.close()

    #pdb.set_trace()
    print ('North American: ' + str(cont_count[1]))
    print ('South American: ' + str(cont_count[2]))
    print ('Europe: ' + str(cont_count[3]))
    print ('Africa: ' + str(cont_count[4]))
    print ('Asia: ' + str(cont_count[5]))
    print ('Australia: ' + str(cont_count[6]))
    print ('Antartica: ' + str(cont_count[7]))
    print ('Oceanic: ' + str(cont_count[8]))
    print ('Island(s): ' + str(cont_count[9]))
    print ('Not a bird (0): ' + str(cont_count[0]))
                # minus out non-birds, easier to do here then earlier
    #birdsnum = birdsnum - sum(cont_count)
    birdsnum = birdsnum - cont_count[0]
    print ('Num of unique birds:' + str(birdsnum))

# read in the bibliographic database and put it in a dictory stored by the
# refernce number which is the type (book, journal, web) + its number
def read_biblio(biblio, filename):
    biblio_file = open(filename)
    for line in biblio_file:
        line = astrip(line)
                        # replace ampersands with the html version &amp;
        line = line.replace('&', '&amp;')
        if not isline(line): continue
        aline = line.split('\t')
        if 'Books:' in aline[0]:
            type = 'b'
        elif 'Web:' in aline[0]:
            type = 'w'
        elif 'Journals:' in aline[0]:
            type = 'j'
        else:
            index = type + aline[0]
            if 'b' in type:
                if (len(aline) < 4):
                    print ("error in:", aline)
                    continue
                ref = aline[1] +  ', <i>' + aline[2] + '</i>,' + aline[3]
            elif 'w' in type:
                if (len(aline) < 3):
                    print ("error in:", aline)
                    continue
                ref = '<a href="' + aline[1] + '">' + aline[1] + '</a> ' +\
                        aline[2]
            elif 'j' in type:
                if (len(aline) < 5):
                    print ("error in:", aline)
                    continue
                ref = aline[1] + ', "' + aline[2] + '" <i>' + aline[3] + \
                        '</i>,' + aline[4]
            biblio[index] = ref
    biblio_file.close()

#   routine to print the bird pictures, sorted by their age (Juvenile and Adult
#   or breeding status.
def print_birds(outfile, bird_lines, birdname, b_dates,blank_pic):
    age = {'1':'Juvenile', '2':'Breeding', '3':'Non-breeding', '4':'Adult'}
    space = ' '
    start = space*10 + '<div class="highslide-gallery">\n' + space*10 + '<table>\n'
    blankp = blank_pic.replace('25%', '140')

    title = space*10 + '<caption class="captitle">{title}</caption>\n'
            # need to figure out if there are breeding dates and if so, put them
            # in the age field. If there are no breeding dates, dates will have the
            # length of 1.  Need to pad that out.
            # If the breeding dates contain a '#', then the bird has breeding
            # plumage, but I don't know the dates.  Get rid of the '#', so no
            # date will try and be printed
    temp = b_dates.replace('#','')
    dates = temp.split(':')
    if len(dates) < 2: dates.append('')

            # Now add the breeding dates (or blanks if none) to the
            # breeding, non-breeding titles
    if dates[0]: age['2'] = age['2'] + ' (' + dates[0] + ')'
    if dates[1]: age['3'] = age['3'] + ' (' + dates[1] + ')'

    keys = list(bird_lines.keys())
    keys.sort()
    for key in keys:
        #print (birdname + ' = ' + key)
        k = int(key)
        if k > 0 and k < 5:
            header = title.format (title=age[key])
        else:
            header = title.format(title=' ')
        outfile.write(start)
        outfile.write(header)
        blist = bird_lines[key]
        num = len(blist)
        ind = 0
        #pdb.set_trace()
        while ind < num:
            outfile.write(space*10 + '<tr>\n')
            if ind < num:
                outfile.write(blist[ind])
            if ind+1 < num:
                outfile.write(blist[ind+1])
            else:
                outfile.write(blankp)
            if ind+2 < num:
                outfile.write(blist[ind+2])
            else:
                outfile.write(blankp)
            outfile.write(space*10 + '</tr>\n')
            ind += 3
    outfile.write (space*10 + '</table>\n')
    outfile.write (space*10 + '</div>\n')
 
# print out pics, either 3 or 4 columns per row.  Used for
# places (3), favorites (3), recents (3), etc.
def print_pics (n_pic, outfile, pic_lines, start, blank_pic):
    space = ' '
    #pdb.set_trace()
    start = space*10 + '<div class="highslide-gallery">\n' + space*10 + '<table class="small">\n'
    title = space*10 + '<caption class="captitle">{title}</caption>\n'

    num = len(pic_lines)
            # make the input (pic_lines) a multiple of 4
            # to ease column problem in html.  This is mainly
            # true when num of pic_lines is less than 4
    while num % n_pic:
        pic_lines.append(' ')
        num += 1

    outfile.write(start)
    num = len(pic_lines)
    i = 1
    #pdb.set_trace()
    for line in pic_lines:
        if i % n_pic == 1:
            outfile.write(space*10 + '<tr>\n')
        if line == " ":
            outfile.write(blank_pic)
        else:
            outfile.write(line)
        if i % n_pic == 0:
            outfile.write(space*10 + '</tr>\n')
        i += 1
    outfile.write (space*10 + '</table>\n')
    if 'div' in start: outfile.write (space*10 + '</div>\n')
        

# Print out the second and final part of a complete description.
# The main thing to do is the bibliography and the surrounding
# html text.
def process_end(lines, biblio, abird):
    space = ' '
    info1 = space*8 + '<a name="moreinfo"></a>\n'
    pstart= space*10 + '<p>\n'
    pend= space*10 + '</p>\n'
    out = []

    out.append(info1)
    out.append(pstart)

    while (lines):
        i = lines.pop(0)
        if 'Bibliography:' in i: break
        out.append (space*10 + i)

            # now do the references, need to make sure there is no
            # extra spaces, the (b, j, w) are lower case, and it is
            # the key is is in the master bibliography database
            # Also, if it is a web bibliography look for the date
            # accessed

            # add header stuff for bibliography
    #pdb.set_trace()
    out.append(space*10 + i)
    out.append(space*10 + '<ul class="list_text">\n')

    while (lines):
                # always get the first line left
        access_date = ' '
        line1 = lines.pop(0)
        line1 = line1.rstrip('\n')
        line1 = line1.lstrip()
        line2 = line1.split(';')
        i = line2[0]
        if len(line2) > 1: access_date = ', Accessed ' + line2[1]
        if 'w' in i and len(line2) < 2:
            print (abird, ':  No date for web page: ', line2)
        i = i.rstrip()
        i = i.lower()
        #pdb.set_trace()
        if '</p>' in i:
            break
        elif 'w' in i or 'b' in i or 'j' in i:
            if (i in biblio):
                out.append(space*10 + '<li>' + biblio[i] + access_date + '</li>\n')
            else:
                print ('erroneous key: ' + i + ' for:' + abird)
        else:
            print ('erroneous key: ' + i + ' for:' + abird)

    out.append (space*10 + '</ul>\n')
    out.append (space*10 + '</p>\n')
    out.append(pend)
    return out

# routine for checking out if a video, feet, or tongue are available
def find_extra (bird, abird, blist, tx_list, btype):
    space = ' '
    text = {}
            # first see if there is any text for that bird and put
            # it into a dictionary based on the extension ('1', '2')
            # otherwise, create a text line to go with the video
    #pdb.set_trace()
    if tx_list:
        for line in tx_list:
            fields = line.split(':')
            text[fields[0]] = fields[1]
        if btype == 'videos': btype='film'
    else:
        if btype == 'video':
            tx = '(video)'
            btype = 'film'
        elif btype == 'tongue':
            tx = '(tongue)'
        elif btype == 'feet':
            tx = '(feet)'

            # cycle through the items found in the 'ls'
    aline = ""
    for f in blist:
                    # get extension number
        i = f.find('_')
        ind = f[i+1]
                    # get rid of the leading '../'
        link = f[3:]
        if ind in text:
            tx_line = text[ind]
        else:
            tx_line = tx
        line = space*10 + '<li><a class="nounder" href="' + link + '">'+ \
            '<img src="Icons/' + btype + '.png" alt="feet" />\n' + \
            space*10 + tx_line + '</a></li>\n'
        aline = aline + line
    return aline

def shorten (aline):
# Simple routine to make a string shorter (for family titles on bird pages) by getting
# rid of ending words and adding ' etc.'
    bline = aline
    #pdb.set_trace()
    if len(aline) < 30: return bline

    while len(bline) > 25:
        beg = bline.rfind(' ')
        bline = bline[0:beg]
    bline = bline + ", etc."
    return bline

#
# main routine
#
alltitles = []      # holds all the titles names, used on class pages for next/prev.
birds = {}          # dictionary for birds
places = {}         # dictionary for places 
favorites = []      # favorite birds to process
num_pixels = {}	    # hold pictures width, height. Indexed by date and pic name (201404:IMG_1234)
recents = {}        # recent bird photos
right_menu={}       # right menu (all birds or places in a class)
index={}            # dictionary holding the bird index information
biblio={}           # bibliography database
breeding={}         # breeding/non-breeding dates where appropriate
blength={}          # dictionary containing length of birds
bwingspan={}        # dictionary containing wingspan of birds
bweight={}          # dictionary containing weight of birds
beggs={}            # dictionary containing clutch size
bchicks={}          # dictionary containing chicks at birth (altricial, precocial)
bstatus={}          # dictionary containing status of birds
tx_tongue={}        # storing text information for tongues
tx_feet={}          # storing text information for feets
# stores in a list the birds found on each continent.
# A bird can be found in multiple continents.
dummy = []          # place holder for places
class_in = {}       # holds the title (Family) and bird/places in it.
class_dic = {}      # maps a bird/place to the appropriate classification
                    # which then used to acces the correct right menu.
tx_video={}         # storing videos information
space = ' '
scien = {}          # Dictionary where scientific name points to a bird (for index)
scien_name = {}     # Dictionary where birdname points to scientifc name (for bird page)

                    # Define and initialize continent which stores for each
                    # continent all the birds found there.
                    # A bird can be found in multiple continents.
cont_mappings= ['N/A', 'NA', 'SA', 'EU', 'AF', 'AS', 'OC', 'AN', 'Oceanic', 'Islands']
continent = {}	    # dictionary to hold the continents for each bird
cont_count = [0,0,0,0,0,0,0,0,0,0]
                    # Internal link to go back to the top of the page
totop = '<p class="top-one"><a href="#atop"><b>Top of Page </b></a></p>\n'
atop =  '<a name="atop" id="atop"></a>\n'

# read in all the databases
read_place_class (right_menu, class_in, class_dic)
read_bird_facts (blength, bwingspan, bweight, beggs, bchicks, bstatus, breeding)
read_bird_class (right_menu, class_in, index, class_dic, alltitles, continent,cont_count,scien, scien_name)

read_data (birds, "inventory.xlsx", "inventory", favorites, recents,num_pixels)
read_data (places, "place.xlsx", "place", dummy, dummy,num_pixels)
read_text (tx_video, "misctext/videos.txt")
read_text (tx_tongue, "misctext/tongues.txt")
read_text (tx_feet, "misctext/feet.txt")
read_biblio (biblio, "misctext/bibliography.txt")

# get a list of videos, feet, tongue, Descriptions, and short text for birds
# Text is optional for the tongue and feet.  Also, complete description are
# gradually replacing short text (dir: text).
ls_text = str(subprocess.check_output(['ls', '../text/*.txt']), encoding='utf8')
ls_desc = str(subprocess.check_output(['ls', '../Descriptions/*.txt']), encoding='utf8')
ls_videos = str(subprocess.check_output(['ls', '../videos/*.mp4']), encoding='utf8')
ls_feet = str(subprocess.check_output(['ls', '../feet/*.jpg']), encoding='utf8')
ls_tongue = str(subprocess.check_output(['ls', '../Tongue/*.jpg']), encoding='utf8')
ls_place = str(subprocess.check_output(['ls', '../text_place/*.txt']), encoding='utf8')
ls_class = str(subprocess.check_output(['ls', '../text_class/*.txt']), encoding='utf8')
ls_title = str(subprocess.check_output(['ls', '../text_title/*.txt']), encoding='utf8')

# covert the lists in videos, feet & tongue to a dictionary
#dic_videos = lstodic(ls_videos)
dic_videos = {}
dic_feet = lstodic(ls_feet)
dic_tongue = lstodic(ls_tongue)

# read in the various html lines used to create the pages
page_head = get_html("html_templates/header.html")
page_end = get_html("html_templates/end.html")
l_menu = get_html("html_templates/l_menu.html")

#head_bird = get_html("html_templates/bird_head.html")
#end_bird =  get_html("html_templates/bird_end.html")
#other_end = get_html("html_templates/other_end.html")
#ind_line = get_html("html_templates/bird_index.html")
#ind_head = get_html("html_templates/index_head.html")
#ind_end = get_html("html_templates/index_end.html")
#class_head = get_html("html_templates/class_head.html")
#class_end = get_html("html_templates/class_end.html")

bird_pic = get_html("html_templates/pic.html")
class_pic = get_html("html_templates/class_pic.html")
blank_pic = get_html("html_templates/blank_pic.html")
video_line = get_html("html_templates/video_line.html")
video_txt = get_html("html_templates/video_text.html")
biblio_txt = get_html("html_templates/biblio_text.html")

# now start creating the web pages in this order, birds, places,
# videos, favorites, latest, bibliography and class pages
keys = list(birds.keys())
for bird in keys:
    atext=[]
    dtext=[]
    head_bird = page_head + l_menu
    ahead = head_bird.replace('{bird}', bird)
    ahead = head_bird.replace('{title}', 'Pictures and information on ' + bird)
    ahead = ahead.replace('{menu}', 'birds_menu.html')
                # bird name without any spaces
    abird = bird.replace(' ', '')
    filename = '../bird_' + abird + '.shtml'
    outfile = open(filename, 'w')
    outfile.writelines(ahead)
    outfile.write(space*12 + '<section id="content">\n')
    outfile.write(space*15 + '<article>\n')
    outfile.write(space*19 + atop)
    tbird =  bird
    if bird in scien_name: tbird = bird + ' &#45; <i>' + scien_name[bird] + '</i>'
    outfile.write(space*19 + '<h3>' + tbird + '</h3>\n')
                # put out the various statistics
    outfile.write(space*19 + '<div id="stats_left">\n')
    outfile.write(space*19 + '<table class="stats">\n')
    outfile.write(space*21 + '<colgroup>\n')
    outfile.write(space*23 + '<col style="width:35%">\n')
    outfile.write(space*23 + '<col style="width:65%">\n')
    outfile.write(space*21 + '</colgroup">\n')
    outfile.write(space*21 + '<tr>\n')
    if bird in blength:
        outfile.write(space*21 + '<tr>\n')
        outfile.write(space*23 + '<td>Length</td>\n')
        outfile.write(space*23 + '<td> <strong1>' + blength[bird] + '</strong1></td>\n')
        outfile.write(space*21 + '</tr>\n')
    if bird in bwingspan:
        outfile.write(space*21 + '<tr>\n')
        outfile.write(space*23 + '<td>Wingspan</td>\n')
        outfile.write(space*23 + '<td> <strong1>' + bwingspan[bird] + '</strong1></td>\n')
        outfile.write(space*21 + '</tr>\n')
    if bird in bweight:
        outfile.write(space*21 + '<tr>\n')
        outfile.write(space*23 + '<td>Weight</td>\n')
        outfile.write(space*23 + '<td> <strong1>' + bweight[bird] + '</strong1></td>\n')
        outfile.write(space*21 + '</tr>\n')
    if bird in beggs:
        outfile.write(space*21 + '<tr>\n')
        outfile.write(space*23 + '<td>Clutch Size</td>\n')
        outfile.write(space*23 + '<td> <strong1>' + beggs[bird] + '</strong1></td>\n')
        outfile.write(space*21 + '</tr>\n')
    if bird in bchicks:
        outfile.write(space*21 + '<tr>\n')
        outfile.write(space*23 + '<td>Chicks at birth</td>\n')
        outfile.write(space*23 + '<td> <strong1>' + bchicks[bird] + '</strong1></td>\n')
        outfile.write(space*21 + '</tr>\n')
    if bird in bstatus:
        outfile.write(space*21 + '<tr>\n')
        outfile.write(space*23 + '<td>IUCN Conservation Status</td>\n')
        if 'Endangered' in bstatus[bird] or 'Extinct' in bstatus[bird] or 'Vulnerable' \
                in bstatus[bird] or 'Threatened' in bstatus[bird]:
            outfile.write(space*23 + '<td> <strong2>' + bstatus[bird] + '</strong2></td>\n')
        else:
            outfile.write(space*23 + '<td> <strong1>' + bstatus[bird] + '</strong1></td>\n')

        outfile.write(space*21 + '</tr>\n')
			# now do the continents
    if bird in continent:
        #pdb.set_trace()
        conts = continent[bird].split('+')
        astring = ''
        for s_conts in conts:
            i_conts = int(s_conts)
            astring += cont_mappings[i_conts] + ', '
        astring = astring[:-2]      # remove the last comma and space
        outfile.write (space*19 + '<td>Continents:</td>')
        outfile.write ('<td><strong1>' + astring + '</strong1></td>\n')

		# following line for debug purposes, puts out the continent
		# numbers above the bird statistics
    #outfile.write(space*19 + continent[bird] + '\n')
    outfile.write(space*19 + '</table>\n')
    outfile.write(space*19 + '</div> <!-- end stats-left -->\n')
    #pdb.set_trace()

                                # now add stuff for videos, feet and tongue if available
    outv = outf = outt = ""
    if abird in dic_videos:     # are there any videos
            outv = find_extra (bird, abird, dic_videos[abird], "", 'video')
    if abird in dic_feet:       # are there any pictures of feet
            outf = find_extra (bird, abird, dic_feet[abird], "", 'feet')
    if abird in dic_tongue:     # are there any pictures of tongues
            outt = find_extra (bird, abird, dic_tongue[abird], "", 'tongue')

    if (outv or outf or outt):
        outfile.write(space*19 + '<div id=stats_right>\n')
        outfile.write(space*21 + '<strong1>Videos/Pictures</strong1>\n')
        outfile.write(space*21 + '<ul class=list_style>\n')
        outfile.writelines(outv)
        outfile.writelines(outf)
        outfile.writelines(outt)
        outfile.write(space*21 + '</ul>\n')
        outfile.write(space*19 + '</div> <!--stats_right -->\n')
    else:
        outfile.write(space*19 + '<div id=stats_right>\n')
        outfile.write(space*21 + '<ul class=list_style>\n')
        outfile.write(space*21 + '&nbsp;\n')
        outfile.write(space*21 + '</ul>\n')
        outfile.write(space*19 + '</div> <!--stats_right -->\n')
    outfile.write(space*19 + '<div style="clear:both;"></div>\n')

                # first find and print out the first part of the description
                # there are three possibilities, complete description in
                # directory 'Description', a partial description in directory
                # 'text', or nothing.
    if abird in ls_desc:
                # a full description is divided into two sections, the
                # first is printed before the pictures, the second after
                # the pictures.  Up until the ':--' or Diet: is the 
                # first part.  If there is a complete description skip
                # the short one in ls_text
        filename = '../Descriptions/' + abird + '.txt'
        file_d = open(filename)
        for aline in file_d:
                        # :--- breaks up the Description to before and after pics
            if ':---' in aline: break
            if isline(astrip(aline)):
                        # modify the <p> for the following text.
                if '<p>' in aline:
                    aline = aline.replace('<p>', '<p class="p_desc p_desc1">')
                outfile.write(space*10 + aline)
                # add the internal link to the second part of the description.
        outfile.write('<p class="p_desc"><a href="#moreinfo">(For more information)</a></p>\n')

                # save the second part for later printing
        for aline in file_d:
            if isline(astrip(aline)):
                if aline[0] == ':':
                    aline = aline.rstrip('\n')
                    aline = aline[1:]
                    aline = '<b>' + aline + '</b>\n'
                elif '<p>' in aline:
                    aline = aline.replace('<p>', '<p class="p_desc p_desc1">')
                atext.append(aline)
        dtext = process_end (atext, biblio, abird)
        #pdb.set_trace()
        file_d.close()

    elif abird in ls_text:      # short description
        file_d = open('../text/' + abird + '.txt')
        flag = 1
        for line in file_d:
                        # if the first <p>, then end the paragraph for the
                        # latin name and modify the <p> for the following text.
            if '<p>' in line:
                if flag:
                    outfile.write(space*10 + '</p>\n')
                    flag = 0
                line = line.replace('<p>', '<p class="p_desc p_desc1">')
            outfile.writelines(space*10 + line)
        file_d.close()

    else:                       # no description
        outfile.write(space*10 + '</p>')
        outfile.write(space*10 + '<p class="p_desc p_desc1">Pictures I have taken of: ' + \
                abird + '.</p>\n')

                # not do the pictures, first sort each picture into its
                # proper category, juvenile, breeding, non-breeding or adult
                # not all categories will have pictures
    blist = birds[bird]
    blist.sort()
    num = 0
                # I use this to decide if I need to add extra space and the
                # end of the pics.  For example, 3 or less pictures means that the
                # background will not extend and cover the left menu.
    numpics = len(blist)
                # cycle through the bird pictures, formatting them and putting
                # them in bird_lines depending on their age (Juvenile, adult, etc)
    #pdb.set_trace()
    bird_lines = {}
    for line in blist:
                # call routine to format pic with all the needed html
        #print (line)
        aline = format_pic(line, num, num_pixels, 0)
        num += 1
        if line[0] in bird_lines:
            bird_lines[line[0]].append(aline)
        else:
            bird_lines[line[0]] = []
            bird_lines[line[0]].append(aline)

                # find out if there are breeding dates
                # now print the bird images
    if bird in breeding:
        print_birds(outfile, bird_lines, bird, breeding[bird],blank_pic)
    else:
        #pdb.set_trace()
        #print ('error in breeding key for : ' + bird)
        print_birds(outfile, bird_lines, bird, " ", blank_pic)
    if dtext:
                # print out the rest of the complete description
        outfile.writelines(dtext)
        outfile.write (space*10 + totop)
    else:
                # If there only a few pictures, add space to the top of the
                # 'Top of Page' line, so the left menu want fall off the page.
        ttop = totop
        if (numpics < 4):
            ttop = totop.replace('one', 'four')
        outfile.write (space*10 + ttop)
    outfile.write (space*8 + '</article>\n')
    outfile.write (space*8 + '</section>\n')

    menu = right_menu[class_dic[bird]]
    outfile.writelines(menu)
    outfile.writelines(page_end)
    outfile.close()

    #pdb.set_trace()
            # now put out the place pages, will cycle through the places
            # dictionary, printing one page at a time
keys = list(places.keys())
for place in keys:
    head_bird = page_head + l_menu
    ahead = head_bird.replace('{bird}', place)
    ahead = head_bird.replace('{title}', 'Pictures of ' + place)
    ahead = ahead.replace('{menu}', 'places_menu.html')
            # place name without any spaces
    aplace = place.replace(' ', '')
    filename = '../bird_' + aplace + '.shtml'
    outfile = open(filename, 'w')
    outfile.writelines(ahead)
    outfile.write(space*12 + '<section id="content">\n')
    outfile.write(space*15 + '<article>\n')
    outfile.write(space*19 + atop)
    outfile.write(space*19 + '<h3>' + place + '</h3>\n')
                # first print out the various info on each place
    if aplace in ls_place:
        file_d = open('../text_place/' + aplace + '.txt')
        for line in file_d:
            outfile.writelines(space*10 + line)
        file_d.close()
    else:
        outfile.writelines(space*10 + '<p>This page contains pictures I have taken of ' \
                + place + '</p>\n')

            # now format the pictures, putting them into an array before printing
    place_lines = []
    num = 0
    for line in places[place]:
        aline = format_pic (line, num, num_pixels, 0)
        num += 1
        place_lines.append(aline)

    start = ' '
    print_pics (3, outfile, place_lines, start, blank_pic) 

    ttop = totop.replace('one', 'two')
    outfile.write (space*10 + ttop)
    outfile.write (space*8 + '</article>\n')
    outfile.write (space*8 + '</section>\n')

    menu = right_menu[class_dic[place]]
    outfile.writelines(menu)
    outfile.writelines(page_end)
    outfile.close()

            # now do the favorites page
favorites.sort(reverse=True)
head_bird = page_head + l_menu
ahead = head_bird.replace('{bird}','Favorites')
ahead = head_bird.replace('{title}','Favorites Pictures')
ahead = ahead.replace('{menu}', 'birds_menu.html')

filename = '../bird_' + 'Favorites' + '.shtml'
outfile = open(filename, 'w')
outfile.writelines(ahead)
outfile.write(space*12 + '<section id="content_main">\n')
outfile.write(space*15 + '<article>\n')
outfile.write(space*19 + atop)
outfile.write(space*19 + '<h3>Favorites</h3>\n')
outfile.write(space*10 + '<p class="p_desc p_desc1">\n')
outfile.write(space*10 + 'Below are pictures of birds that I took that I especially liked </p>\n')

fav = []
num = 0
for line in favorites:
    aline = format_pic (line, num, num_pixels, 1)
    num += 1
    fav.append(aline)
start = ' '
print_pics (3, outfile, fav, start, blank_pic) 

ttop = totop
if len(favorites) < 4: ttop = ttop.replace('one','four')
outfile.write (space*10 + ttop)
outfile.write (space*8 + '</article>\n')
outfile.write (space*8 + '</section>\n')
outfile.writelines(page_end)

outfile.close()

            # now do the recent page
recl = list(recents.values())
recl.sort()
head_bird = page_head + l_menu
ahead = head_bird.replace('{bird}','Recent Photos')
ahead = head_bird.replace('{title}','Recent Photos')
ahead = ahead.replace('{menu}', 'birds_menu.html')
filename = '../bird_' + 'Latest' + '.shtml'
outfile = open(filename, 'w')
outfile.writelines(ahead)
outfile.write(space*12 + '<section id="content_main">\n')
outfile.write(space*15 + '<article>\n')
outfile.write(space*19 + atop)
outfile.write(space*19 + '<h3>Recent Photos</h3>\n')
outfile.write(space*10 + '<p class="p_desc p_desc1">\n')
outfile.write(space*10 + 'Below are some of the pictures of birds that I have recently taken.</p>\n')

rec = []
num = 0
count = 0
for line in recl:
    if count > 30: break
    aline = format_pic (line, num, num_pixels, 1)
    num += 1
    count += 1
    rec.append(aline)
start = ' '
print_pics (3, outfile, rec, start, blank_pic)

ttop = totop
if len(favorites) < 4: ttop = ttop.replace('one','four')
outfile.write (space*10 + ttop)
outfile.write (space*8 + '</article>\n')
outfile.write (space*8 + '</section>\n')
outfile.writelines(page_end)

outfile.close()

        # now do the bibliography page
head_bird = page_head + l_menu
ahead = head_bird.replace('{bird}','Recent Photos')
ahead = head_bird.replace('{title}', 'Bibliography for Bird Journal')
ahead = ahead.replace('{menu}', 'birds_menu.html')
filename = '../Bibliography' + '.shtml'
outfile = open(filename, 'w')
outfile.writelines(ahead)
outfile.write(space*12 + '<section id="content_main">\n')
outfile.write(space*15 + '<article>\n')
outfile.write(space*19 + atop)
outfile.write(space*19 + '<h3>Recent Photos</h3>\n')
outfile.writelines (biblio_txt)

bib_lines = {}
keys = biblio.keys()
btext = {'b':'<b> Books: </b>\n', 'j':'<b> Journals, Magazines </b>\n', 'w':'<b> Web: </b>\n'}
for k in keys:
    ind = k[0]
    if not ind in bib_lines:
        bib_lines[ind] = []
    bib_lines[ind].append(biblio[k])

alist = ['b', 'j', 'w']
for i in alist:
    outfile.write(space*10 + '<img src="Icons/bird.png" alt="bird">' + \
            btext[i])
    outfile.write(space*10 + '<ul class="circle">\n')
    blist = []
    blist = bib_lines[i]
    blist.sort()
    for line in blist:
        outfile.write(space*10 + '<li>' + line + '</li>\n')
    outfile.write(space*10 + '</ul>\n')

        # constant but growing number of entries, so I don't need to worry about
        # bibliography being to few lines
outfile.write (space*10 + totop)
outfile.write (space*8 + '</article>\n')
outfile.write (space*8 + '</section>\n')
outfile.writelines(page_end)
outfile.close()

        # now do the video page
head_bird = page_head + l_menu
ahead = head_bird.replace('{bird}','Bird Videos')
ahead = head_bird.replace('{title}', 'Videos of Birds')
ahead = ahead.replace('{menu}', 'birds_menu.html')

filename = '../videos' + '.shtml'
outfile = open(filename, 'w')
outfile.writelines(ahead)
outfile.write(space*12 + '<section id="content_main">\n')
outfile.write(space*15 + '<article>\n')
outfile.write(space*19 + atop)
video_txt = video_txt.replace('<p>', '<p class="p_desc p_desc1">')
outfile.writelines(video_txt)
outfile.write(space*19 + '<div class="video_tx">\n')
outfile.write(space*19 + '<table>\n')
keys = list(tx_video.keys())
keys.sort()
for k in keys:
    bname = k.replace(' ', '')
    for i in tx_video[k]:
        fields = i.split(':')
        vname = 'videos/' + bname + '_' + fields[0] + '.mp4'
        line = video_line.format(bird=k, video=vname, text= fields[1])
        outfile.write(line)
outfile.write(space*19 + '</table>\n')
outfile.write(space*19 + '</div>\n')
        # constant but growing number of entries, so I don't need to worry about
        # video page being to few lines
outfile.write (space*10 + totop)
outfile.write (space*8 + '</article>\n')
outfile.write (space*8 + '</section>\n')
outfile.writelines(page_end)
outfile.close()

        # now do the bird index page
head_bird = page_head
#pdb.set_trace()
ahead = head_bird.replace('{bird}','Bird Index')
ahead = head_bird.replace('{title}', 'Index of Birds')
filename = '../class_index.shtml'
outfile = open(filename, 'w')
outfile.writelines(ahead)

outfile.write(space*12 + '<br/><p class=index_gr>**Names in green are alternate bird names</p><br/>')
outfile.write(space*12 + '<section class="indexa">\n')
outfile.write(space*15 + '<article>\n')
outfile.write(space*19 + atop)
outfile.write(space*19 + '<table>\n' + space*19 + '<tr>\n')

        # put the index into a hash array depending on the first
        # letter of the last name
            # format for alpha links at the top
line1 = space*19 + '<td class="indexb"><a href="#{letter}"><b class="indexc">{letter}</b></a></td>\n'
            # format for each alpha link (e.g., a, c, x)
line2 = space*4 + \
        '<td style="vertical-align:top;"><a name="{letter}">' + \
        '<b style="color:magenta;">{letter}</b></a></td>\n' + space*4 + '</tr>\n'
            # format for the actual bird index with link
line3 = space*6 + \
        '<td style="vertical-align:top;">\n' + \
        space*6 + \
        '<a href="{link}">{bird}</a> </td>\n'
line3a = space*6 + \
        '<td style="vertical-align:top;">\n' + \
        space*6 + \
        '<a href="{link}">&nbsp;&nbsp;&nbsp;&nbsp;{bird}</a> </td>\n'
            # format for the actual bird index w/o link
line4 = space*6 + '<td style="vertical-align:top;"> {bird} </td>\n'
            # format for a link that uses an alternate bird name
span_s = '<span class="index_gr">'
span_e = '</span>'
        # first sort the bird names into a dictionary based on the first letter
        # of the lastname
alpha = {}
lindex = list(index.keys())
for ind in lindex:
    if not ind[0] in alpha:
        alpha[ind[0]] = []
    alpha[ind[0]].append(ind)

keys = list(alpha.keys())
keys.sort()
        # put out the internal jump index (e.g., 'A', 'B', etc.)
for i in keys:
    linef = line1.format(letter=i)
    outfile.write(linef)

outfile.write(space*19 + '</tr>\n' + space*19 + '</table>\n')

        # now go through each letter of the alphabet first printing the index
        # for that letter followed by the birdnames.
outfile.write(space*4 + '<div class="indext">\n')
outfile.write(space*4 + '<table>\n' + space*4 + '<tr>\n')
for i in keys:
    linef = line2.format(letter=i)
    outfile.write(linef)
    lines = alpha[i]
    lines.sort()
    cols = 4
    num = len(lines)                # total number of birds names to print
    rows = math.ceil(num/cols)      # number of rows needed
        # First go through the letter and process all the files
    out_lines = list()
        # I do this first by going through the array and seeing which are single words,
        # multiple words but unique, multiple words but not unique
    last = ""
    nlines = list()
    for jj in range(0, len(lines)):
        linkname = index[lines[jj]].replace(' ','')
        linkname = 'bird_' + linkname + '.shtml'
        if ',' in lines[jj]:
            (lastnm, fname) = lines[jj].split(',')
            fname = astrip(fname)
            if lastnm == last:                  # output first name only
                if ':1' in fname:
                    fname = fname.replace(':1','')
                    fname = span_s + fname + span_e
                nlines.append(line3a.format(link=linkname, bird=fname))
            elif jj < len(lines)-1:             # output line with last name, followed
                                                # by line with firstname and link
                if (lastnm + ',') in lines[jj+1]:
                    last = lastnm
                    nlines.append(line4.format(bird=lastnm))
                    if ':1' in fname:
                        fname = fname.replace(':1','')
                        fname = span_s + fname + span_e
                    nlines.append(line3a.format(link=linkname, bird=fname))
                else:                           # output full name with link
                    tempname = lines[jj]
                    if ':1' in tempname:
                        tempname = tempname.replace(':1', '')
                        tempname = span_s + tempname + span_e
                    nlines.append(line3.format(link=linkname, bird=tempname))
            else:
                tempname = lines[jj]
                if ':1' in tempname:
                   tempname = tempname.replace(':1', '')
                   tempname = span_s + tempname + span_e
                nlines.append(line3.format(link=linkname, bird=tempname))
        else:
             tempname = lines[jj]
             if ':1' in tempname:
                 tempname = tempname.replace(':1', '')
                 tempname = span_s + tempname + span_e
             nlines.append(line3.format(link=linkname, bird=tempname))
    len_nlines = len(nlines)
    numlines = int(len_nlines/cols)
    if (len_nlines % cols): numlines += 1
    ind = 0
    for kk in range (0, numlines):
        #pdb.set_trace()
        outfile.write (space*4 + '<tr>\n')
        if ind < len_nlines:
            outfile.write (space*4 + nlines[ind])
        if ind+numlines < len_nlines:
            outfile.write (space*4 + nlines[ind+numlines])
        if ind+numlines*2 < len_nlines:
            outfile.write (space*4 + nlines[ind+numlines*2])
        if ind+numlines*3 < len_nlines:
            outfile.write (space*4 + nlines[ind+numlines*3])
        ind += 1
        outfile.write (space*4 + '</tr>\n')

outfile.write(space*19 + '</table>\n')
outfile.write(space*19 + '</div>\n')
outfile.write(space*9 + '</section>\n')
outfile.writelines(page_end)
outfile.close()

        # now do the scientific name index page
head_bird = page_head
#pdb.set_trace()
ahead = head_bird.replace('{bird}','Scientific Name Index')
ahead = head_bird.replace('{title}', 'Index of Scientific Names')
filename = '../class_scien.shtml'
outfile = open(filename, 'w')
outfile.writelines(ahead)

outfile.write(space*12 + '<section class="indexa">\n')
outfile.write(space*15 + '<article>\n')
outfile.write(space*19 + atop)
outfile.write(space*19 + '<table>\n' + space*19 + '<tr>\n')

        # put the index into a hash array depending on the first
        # letter of the last name
            # format for alpha links at the top
line1 = space*19 + '<td class="indexb"><a href="#{letter}"><b class="indexc">{letter}</b></a></td>\n'
            # format for each alpha link (e.g., a, c, x)
line2 = space*4 + \
        '<td style="vertical-align:top;"><a name="{letter}">' + \
        '<b style="color:magenta;">{letter}</b></a></td>\n' + space*4 + '</tr>\n'
            # format for the actual bird index with link
line3 = space*6 + \
        '<td style="vertical-align:top;">\n' + \
        space*6 + \
        '<a href="{link}">{bird}</a> </td>\n'
line3a = space*6 + \
        '<td style="vertical-align:top;">\n' + \
        space*6 + \
        '<a href="{link}">&nbsp;&nbsp;&nbsp;&nbsp;{bird}</a> </td>\n'
            # format for the actual bird index w/o link
line4 = space*6 + '<td style="vertical-align:top;"> {bird} </td>\n'
            # format for a link that uses an alternate bird name
span_s = '<span class="index_gr">'
span_e = '</span>'
        # first sort the bird names into a dictionary based on the first letter
        # of the lastname
alpha = {}
lindex = list(scien.keys())
for ind in lindex:
    if len(ind) < 1: continue
    if not ind[0] in alpha:
        alpha[ind[0]] = []
    alpha[ind[0]].append(ind)

keys = list(alpha.keys())
keys.sort()
        # put out the internal jump index (e.g., 'A', 'B', etc.)
for i in keys:
    linef = line1.format(letter=i)
    outfile.write(linef)

outfile.write(space*19 + '</tr>\n' + space*19 + '</table>\n')

        # now go through each letter of the alphabet first printing the index
        # for that letter followed by the birdnames.
outfile.write(space*4 + '<div class="indext">\n')
outfile.write(space*4 + '<table>\n' + space*4 + '<tr>\n')
        # formating for the individual index page
line3i = space*6 + \
        '<td style="vertical-align:top;">\n' + \
        space*6 + \
        '<a href="{link}">&nbsp;&nbsp;&nbsp;&nbsp;<i>{bird}</i></a> </td>\n'
            # format for the actual bird index w/o link
line4i = space*6 + '<td style="vertical-align:top;"> <i>{bird}</i> </td>\n'
for i in keys:
    linef = line2.format(letter=i)
    outfile.write(linef)
    lines = alpha[i]
    lines.sort()
    cols = 4
    num = len(lines)                # total number of birds names to print
    rows = math.ceil(num/cols)      # number of rows needed
        # First go through the letter and process all the files
    out_lines = list()
        # Scientific names are always at least two words long
        # so I need to group the names with the same first name together
    last = ""
    nlines = list()
    for jj in range(0, len(lines)):
        linkname = scien[lines[jj]].replace(' ','')
        linkname = 'bird_' + linkname + '.shtml'
        #pdb.set_trace()
		    # scientific name can be two or three words long
		    # the first will be fname, the last one or two lastnm
        words = lines[jj].split(' ')
        fname = words[0]
        if len(words) == 2:
            lastnm = words[1]
        elif len(words) > 2:
            lastnm = words[1] + " " + words[2]
        else:
            print (' Two many names in scientific name for:', bird) 
            lastnm = words[1] + words[2]

        lastnm = astrip(lastnm)
        fname = astrip(fname)
        if fname == last:                   # output first name only
            nlines.append(line3i.format(link=linkname, bird=lastnm))
        elif jj < len(lines)-1:             # output line with last name, followed
                                            # by line with firstname and link
            if (fname) in lines[jj+1]:
                last = fname
                nlines.append(line4i.format(bird=fname))
                nlines.append(line3i.format(link=linkname, bird=lastnm))
            else:                           # output full name with link
                nlines.append(line3i.format(link=linkname, bird=lines[jj]))
        else:
            nlines.append(line3i.format(link=linkname, bird=lines[jj]))
    len_nlines = len(nlines)
    numlines = int(len_nlines/cols)
    if (len_nlines % cols): numlines += 1
    if i == 'W' and len_nlines < 4:     # handles a weird case for 'w'
        numlines = 3
    elif i == 'Y' and len_nlines < 4:
        numlines = 3
    ind = 0
    for kk in range (0, numlines):
        #pdb.set_trace()
        outfile.write (space*4 + '<tr>\n')
        if ind < len_nlines:
            outfile.write (space*4 + nlines[ind])
        if ind+numlines < len_nlines:
            outfile.write (space*4 + nlines[ind+numlines])
        if ind+numlines*2 < len_nlines:
            outfile.write (space*4 + nlines[ind+numlines*2])
        if ind+numlines*3 < len_nlines:
            outfile.write (space*4 + nlines[ind+numlines*3])
        ind += 1
        outfile.write (space*4 + '</tr>\n')

outfile.write(space*19 + '</table>\n')
outfile.write(space*19 + '</div>\n')
outfile.write(space*9 + '</section>\n')
outfile.writelines(page_end)
outfile.close()

    # Lastly, print the classification pages
    # each classification page has one title (defines the classification) and option
    # header which group the species
tit_len = len(alltitles)
keys = list(class_in.keys())
keys.sort()
for key in keys:
            # Keep track of the number of pics printed, to see if I need to add
            # extra space at the end to keep the left menu from falling off the
            # page
    numpics=0
    if 'place:' in class_in[key][1]:
            lmenu = 'places_menu.html'
    else:
            lmenu = 'birds_menu.html'
                    # create the next and previous links for top of page
            indx = alltitles.index(key)
            if indx == 0:
                p1 = alltitles[tit_len-1]
            else:
                p1 = alltitles[indx-1]
            prev = class_strip(p1)
            prev = '<b>Previous Group: </b><a href="class_' + prev + '.shtml">' + p1 + '</a>'
            if indx >= (tit_len-1):
                p2 = alltitles[0]
            else:
                p2 = alltitles[indx+1]
            nex = class_strip(p2)
            nex = '<b>Next Group: </b><a href="class_' + nex + '.shtml">' + p2 + '</a>'

    pics = []
    akey = key.replace(' ', '')
    akey = akey.replace(',', '')
    akey = akey.replace('.', '')
    filename = '../class_' + akey + '.shtml'
    outfile = open(filename, 'w')
    head_bird = page_head + l_menu
    ahead = head_bird.replace('{bird}',akey)
    ahead = head_bird.replace('{title}', akey )
    ahead = ahead.replace('{menu}', lmenu)
    outfile.write(ahead)
    outfile.write(space*12 + '<section id="content_main">\n')
    outfile.write(space*15 + '<article>\n')
                    # if a bird class page, write out the next and previous links
    if 'bird:' in class_in[key][1]:
        outfile.write(space*19 + '<div class="rightalign">\n')
        outfile.write(space*21 + prev + '\n')
        outfile.write(space*21 + '<br />')
        outfile.write(space*21 + nex + '\n')
        outfile.write(space*19 + '</div>\n')
    outfile.write('<h1 class="classh">' + key + '</h1>\n')
    outfile.write(space*19 + atop)
            # check to see if there is text for this classification
    if akey in ls_title:
                    # replace the <p> with a class to decrease right margin
        lines = get_html('../text_title/' + akey + '.txt')
        lines = lines.replace('<p>', '<p class="ptext">')
        lines = add_space(lines, '      ')
        outfile.write(lines)
            # now look for sub-classes and print the pics in these subclasses.
            # If there is no sub-class the text in the header line is blank.
    class_pics = []
    flag = 0
    for line in class_in[key]:
        if 'header' in line:
            if (flag):
                print_pics (3, outfile, class_pics, start, blank_pic)
                class_pics = []
            aline = line.replace('header:','')
            if not isline(aline): continue
            bline = aline.replace(' ', '')
            bline = bline.replace(',', '')
            flag = 1
                    # need to find the first word which is used as a anchored link
                    # for the left menu (the anchor is the first world of the header)
            tline = aline.lstrip()
            tline = tline.replace(',', ' ')
            ind = tline.find(' ')
            if ind > -1:
                tline = tline[0:ind]
            outfile.write(space*4 + '<a id="' + tline + '"><h3 class="ptext">' + \
                    astrip(aline) + '</h3></a>\n')
                    # check for text for class
            temp = 'text_class/' + bline + '.txt'
            if temp in ls_class:
                lines = get_html ('../text_class/' + bline + '.txt')
                    # replace the <p> with a class to decrease right margin
                lines = lines.replace('<p>', '<p class="ptext">')
                lines = add_space(lines, '      ')
                outfile.write(lines)
        if 'bird:' in line or 'place:' in line:
            numpics += 1
            fields = line.split(':')
            link = 'bird_' + fields[1].replace(' ', '') + '.shtml'
            link = link.replace(',','')
            #pdb.set_trace()
            bline = class_pic.format(link=link, pic=fields[2], name=fields[1])
            class_pics.append(bline)
    print_pics (3, outfile, class_pics, start, blank_pic)
                    # now see if I need to add spacing for bottom of page
    ttop = totop
    if numpics < 4: ttop = ttop.replace('one','four')
    outfile.write (space*10 + ttop)
    outfile.write (space*8 + '</article>\n')
    outfile.write (space*8 + '</section>\n')
    outfile.write(page_end)

    outfile.close()