import_arxiv_2021.py

# Download HTML
import requests
from lxml import html
import codecs



def download_list(page, filename):

    search= ['guido de croon', 'christophe de wagter', 'ewoud smeur', 'sjoerd tijmons', 'julien dupeyroux', 'salua hamaza', 'marija popovic']
    
    url = 'https://arxiv.org/search/?query='

    if page == 0:
        bibf = codecs.open(filename,'w', 'utf-8')
        bibf.write(u'\ufeff')
        bibf.write('# AUTOGENERATED\n')
        bibf.close()

    listofcodes = []
    
    for person in search:

        u = 'https://arxiv.org/search/?searchtype=author&size=200&query=' + person.replace(' ','+')

        bibf = codecs.open(filename,'a', 'utf-8')
        bibf.write('# Import from: ' + u + '\n')


        print('- Page', u)
        
        p = requests.get(u)

        # Get lines with dataset links
        interest = [s.strip("\r\n").strip().split('arXiv:')[1].strip('</a>').strip('</span') for s in p.text.splitlines(True) if 'arXiv:' in s ]

        for i in interest:
            bibf.write('# - ' + i + '\n')
            if not i in listofcodes:
                listofcodes.append(i)
                print(i)
        #for p in interest:
        bibf.close()

    papernr = 0
    for code in listofcodes:

        if True:
        
            pp = code
            print(pp)

            uu = 'https://arxiv2bibtex.org/?format=bibtex&q='+pp

            pa = requests.get(uu)

            start = False
            bib = []
            for s in pa.text.splitlines(True):
                if '</textarea>' in s:
                    break # only 1
                if start:
                    bib.append(s.strip('\r\n'))
                if '<textarea' in s:
                    start = True
            bib.append('\turl = {https://arxiv.org/abs/'+pp+'},')
            bib.append('\tpdf = {https://arxiv.org/pdf/'+pp+'.pdf},')
            bib.append('}')

            bib = '\n'.join(bib).replace('</textarea>','')
                        
            bib = bib.replace('Title = {','\ttitle = {')
            bib = bib.replace('Year = {','\tyear = {')
            bib = bib.replace('Author = {','\tauthor = {')
            bib = bib.replace('Eprint = {','\teprint = {')
            bib = bib.replace('Howpublished = {','\thowpublished = {')
            bib = bib.replace('Doi = {','\tdoi = {')
            #bib = bib.replace('@article','@misc')

            # open and add, in case of error one can continue
            bibf = codecs.open(filename,'a', 'utf-8')
            #bibf.write('# '+str(papernr)+'\n')
            bibf.write('# '+code+ ': ' + uu + '\n\n')
            bibf.write(bib)
            bibf.write('\n\n\n')
            bibf.close()

            papernr += 1

            # continue is at least 1 paper was found.
            #done = False


        

        # debug: stop after 1 page
        #if pageno >= 1:
        #    done = True

     


download_list(0, 'arxiv.bib')