filter_arxiv.py

import bibtexparser

mavlabpapers = {}

# List of published papers
parser = bibtexparser.bparser.BibTexParser(common_strings=True)
with open('pure.bib', encoding="utf8") as bibtex_file:
    bibtex_str = bibtex_file.read()
bib_database = bibtexparser.loads(bibtex_str, parser=parser)

alldoi = []
for p in bib_database.entries:
    if 'doi' in p:
        alldoi.append(p['doi'].lower())

#print(alldoi)

# List of arXiv papers
xparser = bibtexparser.bparser.BibTexParser(common_strings=True)
with open('./arxiv.bib', encoding="utf8") as xbibtex_file:
    xbibtex_str = xbibtex_file.read()
x_database = bibtexparser.loads(xbibtex_str, parser=xparser)

# Store results
arxiv_database = bibtexparser.bibdatabase.BibDatabase()
rest = bibtexparser.bibdatabase.BibDatabase()

print('================================================================')

for b in x_database.entries:
    hasbeenpublished = False
    # If arxiv version has a  DOI
    if  'doi' in b:
        # check in entire PURE database if the DOI is known
        if b['doi'].lower() in alldoi:
            hasbeenpublished = True

    if not hasbeenpublished:
        arxiv_database.entries.append(b)
    else:
        rest.entries.append(b)

print(mavlabpapers)
    

# dump back
writer = bibtexparser.bwriter.BibTexWriter()
writer.indent = '\t'     # indent entries with 4 spaces instead of one
writer.order_entries_by = 'year'
writer.align_values = True
with open('arxiv_nopub.bib', 'w', encoding='utf8') as bibfile:
    bibfile.write('# AUTOGENERATED\n# \n\n')
    bibfile.write(writer.write(arxiv_database).replace('&',r'\&'))


writer = bibtexparser.bwriter.BibTexWriter()
writer.indent = '\t'     # indent entries with 4 spaces instead of one
writer.order_entries_by = 'year'
writer.align_values = True
with open('arxiv_pub.bib', 'w', encoding='utf8') as bibfile:
    bibfile.write('# AUTOGENERATED\n# \n\n')
    bibfile.write(writer.write(rest).replace('&',r'\&'))
    
print('Done')