Coverage for pybtexris/parsers.py: 100.00%

1import re 

2from collections import defaultdict 

3from pathlib import Path 

4from pybtex import database 

5from pybtex.database.input import BaseParser 

6from pybtex.database import BibliographyData, Entry, Person 

7import csv 

8import unicodedata 

9 

10 

11def clean_entry_key(entry_key): 

12 """ 

13 ensure that the entry key is only in ASCII encoding 

14 """ 

15 return unicodedata.normalize('NFKD', entry_key).encode('ASCII', 'ignore').decode("ascii") 

16 

17 

18def get_entry_key(entry): 

19 """ 

20 Builds an entry key from an entry 

21 """ 

22 people = [x[0] for x in entry.persons.values()] 

23 if people: 

24 first_author = people[0] 

25 entry_key = "-".join(first_author.last_names).replace(" ", ".") 

26 elif "title" in entry.fields: 

27 TRUNCATE_TITLE = 20 

28 entry_key = entry.fields["title"].replace(" ", ".")[:TRUNCATE_TITLE] 

29 else: 

30 entry_key = "Unknown" 

31 

32 if "year" in entry.fields: 

33 entry_key += entry.fields["year"] 

34 

35 entry_key = clean_entry_key(entry_key) 

36 return entry_key 

37 

38 

39data_dir = Path(__file__).parent / "data" 

40 

41 

42class RISParser(BaseParser): 

43 """ 

44 Parser for RIS files. 

45 """ 

46 

47 default_suffix = '.ris' 

48 unicode_io = True 

49 

50 def __init__(self, *args, **kwargs): 

51 super().__init__(*args, **kwargs) 

52 self.ris_type_to_bibtex = {} 

53 self.ris_type_description = {} 

54 

55 with open(data_dir / "types.csv") as f: 

56 reader = csv.reader(f, delimiter=',') 

57 

58 # skip the header 

59 next(reader, None) 

60 for row in reader: 

61 ris_types = row[0] 

62 description = row[1] 

63 bibtex_type = row[2] 

64 

65 for ris_type in ris_types.split(", "): 

66 self.ris_type_to_bibtex[ris_type] = bibtex_type 

67 self.ris_type_description[ris_type] = description 

68 

69 def parse_stream(self, stream): 

70 text = stream.read() 

71 return self.parse_string(text) 

72 

73 def process_entry(self, entry_text): 

74 """ 

75 https://github.com/aurimasv/translators/wiki/RIS-Tag-Map 

76 """ 

77 # Read entry into dictionary 

78 ris_dict = defaultdict(list) 

79 for line in entry_text.split('\n'): 

80 m = re.match(r"^([A-Z0-9]{2})\s*-\s*(.*)$", line.strip()) 

81 if not m: 

82 continue 

83 

84 code = m.group(1) 

85 value = m.group(2) 

86 

87 ris_dict[code].append(value) 

88 

89 # Read type of entry 

90 ris_type = ris_dict.pop("TY", ["GEN"]) 

91 ris_type = ris_type[0] 

92 ris_description = None 

93 if ris_type in self.ris_type_description.values(): 

94 index = list(self.ris_type_description.values()).index(ris_type) 

95 ris_description = ris_type 

96 ris_type = list(self.ris_type_description.keys())[index] 

97 

98 ris_type = ris_type.upper() 

99 if not ris_description: 

100 ris_description = self.ris_type_description[ris_type] 

101 

102 # Create Entry object 

103 bibtex_type = self.ris_type_to_bibtex.get(ris_type, "misc") 

104 entry = Entry(bibtex_type) 

105 entry.fields["type"] = ris_description 

106 

107 # Read People 

108 def add_person(code, role): 

109 names = ris_dict.pop(code, []) 

110 for name in names: 

111 person = Person(name) 

112 entry.add_person(person, role) 

113 

114 add_person("AU", "author") 

115 add_person("A1", "author") 

116 editor_role = [ 

117 "ANCIENT", 

118 "BLOG", 

119 "BOOK", 

120 "CHAP", 

121 "CLSWK", 

122 "COMP", 

123 "DATA", 

124 "CPAPER", 

125 "CONF", 

126 "DICT", 

127 "EDBOOK", 

128 "EBOOK", 

129 "ECHAP", 

130 "ENCYC", 

131 "MAP", 

132 "MUSIC", 

133 "MULTI", 

134 "RPRT", 

135 "SER", 

136 "UNPB", 

137 "ELEC", 

138 ] 

139 add_person("A2", "editor" if ris_type in editor_role else "author") 

140 editor_role += ["ADVS", "SLIDE", "SOUND", "VIDEO"] 

141 add_person("A3", "editor" if ris_type in editor_role else "author") 

142 add_person("A4", "editor" if ris_type in editor_role else "author") 

143 add_person("ED", "editor") 

144 

145 # Read Other Fields 

146 def add_field(code, bibtex_field, delimiter=", "): 

147 values = ris_dict.pop(code, []) 

148 for value in values: 

149 if bibtex_field in entry.fields: 

150 entry.fields[bibtex_field] += f"{delimiter}{value}" 

151 else: 

152 entry.fields[bibtex_field] = value 

153 

154 add_field("TI", "title") 

155 if "title" not in entry.fields: 

156 add_field("T1", "title") 

157 add_field("JO", "journal") 

158 

159 bibtex_t2 = None 

160 if ris_type in ["ABST", "INPR", "JFULL", "JOUR", "EJOUR"]: 

161 bibtex_t2 = "journal" 

162 elif ris_type in ["ANCIENT", "CHAP", "ECHAP"]: 

163 bibtex_t2 = "booktitle" 

164 elif ris_type in ["BOOK", "CTLG", "CLSWK", "COMP", "DATA", "MPCT", "MAP", "MULTI", "RPRT", "UNPB", "ELEC"]: 

165 bibtex_t2 = "series" 

166 if bibtex_t2: 

167 add_field("T2", bibtex_t2) 

168 

169 bibtex_bt = bibtex_t2 

170 if ris_type == "BOOK": 

171 bibtex_bt = "title" 

172 if bibtex_bt: 

173 add_field("BT", bibtex_bt) 

174 

175 bibtex_t3 = None 

176 if ris_type in [ 

177 "BOOK", 

178 "CTLG", 

179 "CLSWK", 

180 "COMP", 

181 "DATA", 

182 "MPCT", 

183 "MAP", 

184 "MULTI", 

185 "RPRT", 

186 "UNPB", 

187 "ELEC", 

188 "ADVS", 

189 "SLIDE", 

190 "SOUND", 

191 "VIDEO", 

192 "CHAP", 

193 "CONF", 

194 "DATA", 

195 "EBOOK", 

196 "ECHAP", 

197 "GOVDOC", 

198 "MUSIC", 

199 "SER", 

200 ]: 

201 bibtex_t3 = "series" 

202 if bibtex_t3: 

203 add_field("T3", bibtex_t3) 

204 

205 add_field("PB", "publisher") 

206 add_field("CY", "address") 

207 add_field("IS", "number") 

208 add_field("DO", "doi") 

209 add_field("VL", "volume") 

210 add_field("SP", "pages") 

211 add_field("UR", "url") 

212 add_field("KW", "keywords", delimiter=" | ") 

213 add_field("N1", "note", delimiter=" | ") 

214 add_field("N2", "note", delimiter=" | ") 

215 add_field("PY", "year") 

216 add_field("AB", "abstract") 

217 if "year" not in entry.fields: 

218 add_field("Y1", "year") 

219 

220 # Read ISBN or ISSN 

221 serial_numbers = ris_dict.pop("SN", []) 

222 for serial_number in serial_numbers: 

223 sn_digits = re.sub(r"\D", "", serial_number) 

224 sn_field = "issn" if len(sn_digits) == 8 else "isbn" 

225 entry.fields[sn_field] = serial_number 

226 

227 entry_key = ris_dict.pop("ID", [None]) 

228 entry_key = entry_key[0] 

229 

230 # Check if DA field could be a month 

231 dates = ris_dict.get("DA", []) 

232 for date in dates: 

233 digital_month = date.isdigit() and (1 <= int(date) <= 12) 

234 text_month = date[:3].lower() in [ 

235 "jan", 

236 "feb", 

237 "mar", 

238 "apr", 

239 "may", 

240 "jun", 

241 "jul", 

242 "aug", 

243 "sep", 

244 "oct", 

245 "nov", 

246 "dec", 

247 ] 

248 if digital_month or text_month: 

249 entry.fields["month"] = date 

250 ris_dict.pop("DA") 

251 

252 end_pages = ris_dict.pop("EP", []) 

253 for end_page in end_pages: 

254 if "pages" in entry.fields: 

255 entry.fields["pages"] += f"--{end_page}" 

256 else: 

257 entry.fields["pages"] = end_page 

258 

259 # Add the remaining fields with the RIS code as the field name 

260 ris_dict.pop("ER", None) 

261 for code, values in ris_dict.items(): 

262 entry.fields[code] = ", ".join(values) 

263 

264 if not entry_key: 

265 entry_key = get_entry_key(entry) 

266 else: 

267 entry_key = clean_entry_key(entry_key) 

268 

269 return entry_key, entry 

270 

271 def parse_string(self, text): 

272 self.unnamed_entry_counter = 1 

273 self.command_start = 0 

274 

275 entry_texts = re.split(r"ER\s+-", text) 

276 entries = (self.process_entry(t) for t in entry_texts if t.strip()) 

277 self.data.add_entries(entries) 

278 return self.data 

279 

280 

281class SuffixParser(BaseParser): 

282 """ 

283 A parser which chooses the parser based on the suffix of each file given to it. 

284 """ 

285 

286 def parse_file(self, filename, file_suffix=None): 

287 if file_suffix is not None: 

288 filename = str(filename) + file_suffix 

289 

290 file_data = database.parse_file(filename) 

291 self.data.add_entries(file_data.entries.items()) 

292 if file_data._preamble: 

293 self.data._preamble.extend(file_data._preamble) 

294 

295 return file_data