import_dataverse_2021.py

# Download HTML
import requests
from lxml import html
import codecs
from habanero import cn


import bibtexparser


# create a bibtex object to add bibtex entries
bib_database = bibtexparser.bibdatabase.BibDatabase()


#from requests_html import HTMLSession

def download_list(page, filename):

    url = 'https://dataverse.nl/dataverse/mavlab'

    if page == 0:
        bibf = codecs.open(filename,'w', 'utf-8')
        bibf.write(u'\ufeff')
        bibf.write('# AUTOGENERATED\n# Import from: '+url+'\n\n\n')
        bibf.close()

    # add some text to the bib_database


    papernr = 1
    pageno = page
    if True:
        print('- Page',pageno)
        
        p = requests.get(url) # + '&page=%d' % pageno)
        print('Downloaded...\n')

        # Download dataverse page
        dom = html.fromstring(p.text.encode('utf-8'))

        # Get lines with dataset links
        interest = [s.strip("\r\n").strip().strip('<a href="').split('"')[0] for s in p.text.splitlines(True) if '/dataset.xhtml?persistentId=doi:' in s and not 'icon-dataset' in s]

        #parser = etree.HTMLParser(recover=True)
        #tree = etree.fromstring(url, parser=parser)
        for p in interest:
            doitxt = p.strip('/dataset.xhtml?persistentId=doi:')
            print(doitxt)

            bib = cn.content_negotiation(ids = doitxt, format = "bibentry")
            bib = bib.replace('}, ', '},\n\t')
            #bib = bib.replace('&quot;', '\"')
            #bib = bib.replace('https://doi.org/','')

            # add bibtex entry to database
            bib_database.entries.append(bib)

            print(bib)          
            # open and add, in case of error one can continue
            bibf = codecs.open(filename,'a', 'utf-8')
            #bibf.write('# '+str(pageno)+', '+str(papernr)+'\n')
            bibf.write('# '+'https://dataverse.nl'+p+'\n\n')
            bibf.write(bib)
            bibf.write('\n')
            bibf.close()

            papernr += 1

            # continue if at least 1 paper was found.
            #done = False
            #break

        pageno += 1

        # debug: stop after 1 page
        #if pageno >= 1:
        #    done = True

    writer = bibtexparser.bwriter.BibTexWriter()
    writer.indent = '\t'     # indent entries with 4 spaces instead of one
    writer.order_entries_by = 'year'
    writer.align_values = True
    with open('test.bib', 'w', encoding='utf8') as bibfile:
        bibfile.write('# AUTOGENERATED\n# \n\n')
        bibfile.write(writer.write(bib_database).replace('&',r'\&'))

     


download_list(0, 'dataverse.bib')