您的位置：首页 > 编程语言 > Python开发

python自动从arxiv下载paper的示例代码

2020-12-09 04:06 1166 查看

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time  : 2020/02/11 21:44
# @Author : dangxusheng
# @Email  : dangxusheng163@163.com
# @File  : download_by_href.py
'''
自动从arxiv.org 下载文献
'''

import os
import os.path as osp
import requests
from lxml import etree
from pprint import pprint
import re
import time
import glob

headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36",
"Host": 'arxiv.org'
}

HREF_CN = 'http://cn.arxiv.org/pdf/'
HREF_SRC = 'http://cn.arxiv.org/pdf/'
SAVE_PATH = '/media/dangxs/E/Paper/download_at_20200730'
os.makedirs(SAVE_PATH, exist_ok=True)

FAIL_URLS = []
FAIL_URLS_TXT = f'{SAVE_PATH}/fail_urls.txt'

def download(url, title):
pattern = r'[\\/:*?"\'<>|\r\n]+'
new_title = re.sub(pattern, " ", title)
print(f'new title: {new_title}')
save_filepath = '%s/%s.pdf' % (SAVE_PATH, new_title)
if osp.exists(save_filepath) and osp.getsize(save_filepath) > 50 * 1024:
print(f'this pdf is be existed.')
return True
try:
with open(save_filepath, 'wb') as file:
# 分字节下载
r = requests.get(url, stream=True, timeout=None)
for i in r.iter_content(2048):
file.write(i)
if osp.getsize(save_filepath) >= 10 * 1024:
print('%s 下载成功.' % title)
return True
except Exception as e:
print(e)
return False

# 从arxiv.org 去下载
def search(start_size=0, title_keywords='Facial Expression'):
# 访问地址: https://arxiv.org/find/grp_eess,grp_stat,grp_cs,grp_econ,grp_math/1/ti:+Face/0/1/0/past,2018,2019/0/1?skip=200&query_id=1c582e6c8afc6146&client_host=cn.arxiv.org
req_url = 'https://arxiv.org/search/advanced'
req_data = {
'advanced': 1,
'terms-0-operator': 'AND',
'terms-0-term': title_keywords,
'terms-0-field': 'title',
'classification-computer_science': 'y',
'classification-physics_archives': 'all',
'classification-include_cross_list': 'include',
'date-filter_by': 'date_range', # date_range | specific_year
# 'date-year': DOWN_YEAR,
'date-year': '',
'date-from_date': '2015',
'date-to_date': '2020',
'date-date_type': 'announced_date_first', # submitted_date | submitted_date_first | announced_date_first
'abstracts': 'show',
'size': 50,
'order': '-announced_date_first',
'start': start_size,
}
res = requests.get(req_url, params=req_data, headers=headers)
html = res.content.decode()
html = etree.HTML(html)

total_text = html.xpath('//h1[@class="title is-clearfix"]/text()')
total_text = ''.join(total_text).replace('\n', '').lstrip(' ').strip(' ')
# i.e. : Showing 1–50 of 355 results
num = re.findall('\d+', total_text)
# Sorry, your query returned no results
if len(num) == 0: return [], 0

total = int(num[-1]) # 查询总条数
paper_list = html.xpath('//ol[@class="breathe-horizontal"]/li')
info_list = []
for p in paper_list:
title = p.xpath('./p[@class="title is-5 mathjax"]//text()')
title = ''.join(title).replace('\n', '').lstrip(' ').strip(' ')
href = p.xpath('./div/p/a/@href')[0]
info_list.append({'title': title, 'href': href})

return info_list, total

# 去指定页面下载
def search_special():
res = requests.get('https://gitee.com/weberyoung/the-gan-zoo?_from=gitee_search')
html = res.content.decode()
html = etree.HTML(html)

paper_list = html.xpath('//div[@class="file_content markdown-body"]//li')
info_list = []
for p in paper_list:
title = p.xpath('.//text()')
title = ''.join(title).replace('\n', '').lstrip(' ').strip(' ')
href = p.xpath('./a/@href')[0]
info_list.append({'title': title, 'href': href})

pprint(info_list)
return info_list

if __name__ == '__main__':
page_idx = 0
total = 1000
keywords = 'Facial Action Unit'
while page_idx <= total // 50:
paper_list, total = search(page_idx * 50, keywords)
print(f'total: {total}')
if total == 0:
print('no found .')
exit(0)

for p in paper_list:
title = p['title']
href = HREF_CN + p['href'].split('/')[-1] + '.pdf'
print(href)
if not download(href, title):
print('从国内镜像下载失败，从源地址开始下载 >>>>')
# 使用国际URL再下载一次
href = HREF_SRC + p['href'].split('/')[-1] + '.pdf'
if not download(href, title):
FAIL_URLS.append(p)
page_idx += 1

# 下载最后的部分
last_1 = total - page_idx * 50
paper_list, total = search(last_1, keywords)
for p in paper_list:
title = p['title']
href = HREF_CN + p['href'].split('/')[-1] + '.pdf'
if not download(href, title):
FAIL_URLS.append(p)
time.sleep(1)

pprint(FAIL_URLS)
with open(FAIL_URLS_TXT, 'a+') as f:
for item in FAIL_URLS:
href = item['href']
title = item['title']
f.write(href + '\n')

print('done.')

以上就是python自动从arxiv下载paper的示例代码的详细内容，更多关于python 从arxiv下载paper的资料请关注脚本之家其它相关文章！

您可能感兴趣的文章:

内容来自用户分享和网络整理，不保证内容的准确性，如有侵权内容，可联系管理员处理

标签： python paper arxiv

相关文章推荐

新的分享

章节导航