begin_line = -1
end_line = 500

with open('Data/Biocatalysis/biocatalysis.txt', 'r') as f:      
    for n,line in enumerate(f):
        if ((n > begin_line) and (n < end_line)):
            print(line.rstrip())
f.close()

with open('Data/Biocatalysis/biocatalysis.txt', 'r') as f:      
    record={}
    corpus={}
    end_of_record = 'SO  '
    unique_id = 'PMID'
    
    for line in f:
        line = line.rstrip()
        tag = line[0:4]
        content = line[6:]

        if (tag == "    "):
            tag = lasttag
            
        lasttag = tag            
        short_tag = tag.rstrip()

        if (len(content)>0):
            if short_tag in record:
                record[short_tag]=record[short_tag]+"; "+content
            else:
                record[short_tag]=content
            
        if (tag == unique_id):
            record = {}
            article_id = content

        if (tag == end_of_record):
            corpus[article_id]=record
            if (len(corpus)%100 == 0):
                print("Stored article number",len(corpus))
                print(record)
                print()

f.close()

import json

with open('Data/Output/biocatalysis_JSON.txt', 'r') as f:      
    data = json.load(f)
f.close()

## Is this the Index file??
with open('Data/Output/biocatalysis_nation.txt', 'r') as f:      
    index = json.load(f)
f.close()

thesauri = {}
with open('Data/country_thesauri.txt', 'r') as f:  
    for line in f:
        line = line.strip()
        entry = line.split(",")
        key = entry[0]
        value = entry[1]
        thesauri[key]=value
f.close()

pub = []
with open('Data/country_pub.txt', 'r') as f:  
    for n,line in enumerate(f):
        line = line.strip()
        entry = line.split(",")
        if (n == 0):
            header = entry
        else:
            pub.append(entry)
            
f.close()

print article

article1 = data['24965656']

print(article1.keys())

import collections as c

journal_counter = c.Counter()
for article_id in data:
    article = data[article_id]
    if ('PL' in article):
        cite=article['PL']
        cite_part = cite.split('. ')
        journal = cite_part[0]
        journal_counter[journal]+=1

new_list=journal_counter.most_common(10)      
print("The top ten nations in this set are:")
for pair in new_list:
    print(pair[0],pair[1])

import pandas as pd

df = pd.DataFrame.from_dict(index, orient='index')
df = df.fillna(0)
df.head()

import json
import numpy as np

with open('Data/Output/biocatalysis_JSON.txt') as fh:
    year_supp = json.load(fh)

sup_lookup = {}
for el in year_supp:
    article_id = el[0]
    year = el[1]
    cite = el[2]
    sup_lookup[article_id] = (year,cite)

import matplotlib.cm as cmx
import matplotlib.colors as colors
pyl.rcParams['figure.figsize'] = (9.0, 7.0)

x4,y4 = X2[:,0],X2[:,1]

plt.suptitle('Biocatalysis Citation Regions', fontsize=14)
plt.hexbin(x4,y4,cite_list,gridsize=nbins,cmap=cmx.rainbow,norm=colors.LogNorm())
plt.colorbar()
plt.show()

import json

with open('Data/Output/biocatalysis_JSON.txt') as fh:
    word_counters = json.load(fh)
len(word_counters)

sample_counter = {}

sample_size = 1000

for n,el in enumerate(word_counters.items()):
    if (n < 1000):
        key = el[0]
        value = el[1]
        sample_counter[key]=value   

##wordcloud package is not supported by MathSageCloud

import matplotlib.pyplot as plt
from wordcloud import WordCloud

n_top_words = 20
wordcloudobj = WordCloud( background_color='white')

for i, topic in enumerate(topic_word):
    topic_words = word_array[np.argsort(topic)][:-n_top_words:-1]
    word_cloud_string =' '.join(topic_words)
    wordcloud = wordcloudobj.generate(word_cloud_string)
    plt.imshow(wordcloud)
    plt.axis("off")
    plt.show()

import json

with open('Data/Output/biocatalysis_nation.txt') as fh:
    bio_nat = json.load(fh)

N = nx.Graph()

for key in bio_nat:
    nat_list = bio_nat[key]
    N.add_nodes_from(bio_nat)
    l = list(it.combinations(nat_list,2))
    if (len(l) > 0):
        for e in l:
            a = e[0]
            b = e[1]
            if (N.has_edge(a,b)):
                N[a][b]['weight'] += 1
            else:
                N.add_edge(a,b,weight=1)