|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
#!/usr/bin/env python# -*- coding: utf-8 -*-# @Time : 2020/02/11 21:44# @Author : dangxusheng# @Email : dangxusheng163@163.com# @File : download_by_href.py'''自动从arxiv.org 下载文献'''import osimport os.path as ospimport requestsfrom lxml import etreefrom pprint import pprintimport reimport timeimport globheaders = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.87 Safari/537.36", "Host": 'arxiv.org'}HREF_CN = 'http://cn.arxiv.org/pdf/'HREF_SRC = 'http://cn.arxiv.org/pdf/'SAVE_PATH = '/media/dangxs/E/Paper/download_at_20200730'os.makedirs(SAVE_PATH, exist_ok=True)FAIL_URLS = []FAIL_URLS_TXT = f'{SAVE_PATH}/fail_urls.txt'def download(url, title): pattern = r'[\\/:*?"\'<>|\r\n]+' new_title = re.sub(pattern, " ", title) print(f'new title: {new_title}') save_filepath = '%s/%s.pdf' % (SAVE_PATH, new_title) if osp.exists(save_filepath) and osp.getsize(save_filepath) > 50 * 1024: print(f'this pdf is be existed.') return True try: with open(save_filepath, 'wb') as file: # 分字节下载 r = requests.get(url, stream=True, timeout=None) for i in r.iter_content(2048): file.write(i) if osp.getsize(save_filepath) >= 10 * 1024: print('%s 下载成功.' % title) return True except Exception as e: print(e) return False# 从arxiv.org 去下载def search(start_size=0, title_keywords='Facial Expression'): # 访问地址: https://arxiv.org/find/grp_eess,grp_stat,grp_cs,grp_econ,grp_math/1/ti:+Face/0/1/0/past,2018,2019/0/1?skip=200&query_id=1c582e6c8afc6146&client_host=cn.arxiv.org req_url = 'https://arxiv.org/search/advanced' req_data = { 'advanced': 1, 'terms-0-operator': 'AND', 'terms-0-term': title_keywords, 'terms-0-field': 'title', 'classification-computer_science': 'y', 'classification-physics_archives': 'all', 'classification-include_cross_list': 'include', 'date-filter_by': 'date_range', # date_range | specific_year # 'date-year': DOWN_YEAR, 'date-year': '', 'date-from_date': '2015', 'date-to_date': '2020', 'date-date_type': 'announced_date_first', # submitted_date | submitted_date_first | announced_date_first 'abstracts': 'show', 'size': 50, 'order': '-announced_date_first', 'start': start_size, } res = requests.get(req_url, params=req_data, headers=headers) html = res.content.decode() html = etree.HTML(html) total_text = html.xpath('//h1[@class="title is-clearfix"]/text()') total_text = ''.join(total_text).replace('\n', '').lstrip(' ').strip(' ') # i.e. : Showing 1–50 of 355 results num = re.findall('\d+', total_text) # Sorry, your query returned no results if len(num) == 0: return [], 0 total = int(num[-1]) # 查询总条数 paper_list = html.xpath('//ol[@class="breathe-horizontal"]/li') info_list = [] for p in paper_list: title = p.xpath('./p[@class="title is-5 mathjax"]//text()') title = ''.join(title).replace('\n', '').lstrip(' ').strip(' ') href = p.xpath('./div/p/a/@href')[0] info_list.append({'title': title, 'href': href}) return info_list, total# 去指定页面下载def search_special(): res = requests.get('https://gitee.com/weberyoung/the-gan-zoo?_from=gitee_search') html = res.content.decode() html = etree.HTML(html) paper_list = html.xpath('//div[@class="file_content markdown-body"]//li') info_list = [] for p in paper_list: title = p.xpath('.//text()') title = ''.join(title).replace('\n', '').lstrip(' ').strip(' ') href = p.xpath('./a/@href')[0] info_list.append({'title': title, 'href': href}) pprint(info_list) return info_listif __name__ == '__main__': page_idx = 0 total = 1000 keywords = 'Facial Action Unit' while page_idx <= total // 50: paper_list, total = search(page_idx * 50, keywords) print(f'total: {total}') if total == 0: print('no found .') exit(0) for p in paper_list: title = p['title'] href = HREF_CN + p['href'].split('/')[-1] + '.pdf' print(href) if not download(href, title): print('从国内镜像下载失败，从源地址开始下载 >>>>') # 使用国际URL再下载一次 href = HREF_SRC + p['href'].split('/')[-1] + '.pdf' if not download(href, title): FAIL_URLS.append(p) page_idx += 1 # 下载最后的部分 last_1 = total - page_idx * 50 paper_list, total = search(last_1, keywords) for p in paper_list: title = p['title'] href = HREF_CN + p['href'].split('/')[-1] + '.pdf' if not download(href, title): FAIL_URLS.append(p) time.sleep(1) pprint(FAIL_URLS) with open(FAIL_URLS_TXT, 'a+') as f: for item in FAIL_URLS: href = item['href'] title = item['title'] f.write(href + '\n') print('done.') |
以上就是python自动从arxiv下载paper的示例代码的详细内容，更多关于python 从arxiv下载paper的资料请关注服务器之家其它相关文章！
原文链接：https://www.cnblogs.com/dxscode/p/13406238.html
