Coverage for src/history/models.py: 57%

1import datetime

2from datetime import timedelta

3from itertools import tee

5from django.db import models

6from django.db.models import JSONField, Q

7from django.db.models.functions import TruncMonth

8from django.utils import timezone

11class HistoryEventQuerySet(models.QuerySet):

12 def get_stale_events(self):

13 distinct_pids = self.order_by("pid", "type").distinct("pid", "type")

14 latest_events = [

15 self.filter(pid=event.pid, type=event.type).latest("created_on").pk

16 for event in distinct_pids

17 ]

18 return self.exclude(pk__in=latest_events)

20 def get_last_unsolved_error(self, pid, strict):

21 if strict:

22 result = self.filter(status="ERROR", pid=pid).latest("created_on")

23 else:

24 result = self.filter(status="ERROR", pid__startswith=pid).latest("created_on")

25 if "obsolete" in result.data:

26 raise HistoryEvent.DoesNotExist

27 return result

30class HistoryEvent(models.Model):

31 created_on = models.DateTimeField(db_index=True, default=timezone.now)

32 type = models.CharField(max_length=200, db_index=True)

33 pid = models.CharField(max_length=64, db_index=True, default="")

34 col = models.CharField(max_length=64, db_index=True, default="")

35 status = models.CharField(max_length=10, db_index=True, default="OK")

36 data = JSONField(default=dict)

37 objects = HistoryEventQuerySet.as_manager()

39 def __str__(self):

40 return f"{self.pid}:{self.type} - {self.created_on}"

42 def is_expandable(self):

43 result = (

44 ("ids_count" in self.data and self.data["ids_count"] > 1)

45 or ("message" in self.data and len(self.data["message"]) > 0)

46 or ("issues" in self.data and len(self.data["issues"]) > 0)

47 )

49 return result

52def test_create():

53 HistoryEvent.objects.all().delete()

55 insert_history_event(

56 {

57 "type": "edit",

58 "pid": "ALCO_2018__1_5_26_0",

59 "col": "ALCO",

60 "status": "OK",

61 "created_on": datetime.datetime(2018, 12, 3, tzinfo=datetime.UTC),

62 "data": {"message": "MR:mr1 false_positive"},

63 }

64 )

66 insert_history_event(

67 {

68 "type": "matching",

69 "pid": "AIF_2015__65_6_2331_0",

70 "col": "AIF",

71 "status": "WARNING",

72 "created_on": datetime.datetime(2018, 12, 10, tzinfo=datetime.UTC),

73 "data": {

74 "message": "MR ne répond pas",

75 "ids_count": 2,

76 "ids": [

77 {"type": "zbl", "id": "zbl1", "seq": 15},

78 {"type": "zbl", "id": "zbl2", "seq": 22},

79 ],

80 },

81 }

82 )

84 insert_history_event(

85 {

86 "type": "matching",

87 "pid": "AIF_2015__65_6_2331_0",

88 "col": "AIF",

89 "status": "OK",

90 "created_on": datetime.datetime(2018, 12, 11, 8, tzinfo=datetime.UTC),

91 "data": {"ids_count": 1, "ids": [{"type": "mr", "id": "mr1", "seq": 7}]},

92 }

93 )

95 insert_history_event(

96 {

97 "type": "edit",

98 "pid": "AIF_2015__65_6_2331_0",

99 "col": "AIF",

100 "status": "OK",

101 "created_on": datetime.datetime(2018, 12, 11, 17, tzinfo=datetime.UTC),

102 "data": {"message": "MR:mr12 checked"},

103 }

104 )

105

106 insert_history_event(

107 {

108 "type": "edit",

109 "pid": "AIF_2009__59_7_2593_0",

110 "col": "AIF",

111 "status": "OK",

112 "created_on": datetime.datetime(2018, 12, 12, 10, tzinfo=datetime.UTC),

113 "data": {"message": "[2] zbl:zbl4 false_positive"},

114 }

115 )

116

117 insert_history_event(

118 {

119 "type": "edit",

120 "pid": "AIF_2009__59_7_2593_0",

121 "col": "AIF",

122 "status": "OK",

123 "created_on": datetime.datetime(2018, 12, 12, 10, tzinfo=datetime.UTC),

124 "data": {"message": "[7] zbl:zbl8 checked"},

125 }

126 )

127

128 insert_history_event(

129 {

130 "type": "edit",

131 "pid": "AIF_2009__59_7_2700_0",

132 "col": "AIF",

133 "status": "OK",

134 "created_on": datetime.datetime(2018, 12, 12, 11, tzinfo=datetime.UTC),

135 "data": {"message": "[3] zbl:zbl5 checked"},

136 }

137 )

138

139 insert_history_event(

140 {

141 "type": "edit",

142 "pid": "AIF_2009__59_7_2611_0",

143 "col": "AIF",

144 "status": "OK",

145 "created_on": datetime.datetime(2018, 12, 12, 15, tzinfo=datetime.UTC),

146 "data": {"message": "All ids checked"},

147 }

148 )

149

150 insert_history_event(

151 {

152 "type": "matching",

153 "pid": "AIF_2007__57_7",

154 "col": "AIF",

155 "status": "ERROR",

156 "created_on": datetime.datetime(2019, 1, 4, tzinfo=datetime.UTC),

157 "data": {

158 "message": "Internal Error",

159 "ids_count": 3,

160 "articles": [

161 {

162 "pid": "AIF_2007__57_7_2143_0",

163 "ids": [{"type": "zbl", "id": "zbl3", "seq": 5}],

164 },

165 {

166 "pid": "AIF_2007__57_7_2143_0",

167 "ids": [

168 {"type": "zbl", "id": "zbl4", "seq": 8},

169 {"type": "mr", "id": "mr2", "seq": 12},

170 ],

171 },

172 ],

173 },

174 }

175 )

176

177 insert_history_event(

178 {

179 "type": "edit",

180 "pid": "AIF_2007__57_7",

181 "col": "AIF",

182 "status": "OK",

183 "created_on": datetime.datetime(2019, 1, 5, tzinfo=datetime.UTC),

184 "data": {"message": "All ids checked"},

185 }

186 )

187

188 insert_history_event(

189 {

190 "type": "edit",

191 "pid": "JEP_2014__1_7_352_0",

192 "col": "JEP",

193 "status": "OK",

194 "data": {"message": "Zbl:zbl8 checked"},

195 }

196 )

197

198 insert_history_event({"type": "deploy", "col": "ALCO", "pid": "ALCO"})

199 insert_history_event({"type": "deploy", "col": "ALCO", "pid": "ALCO_2018__1_1"})

200 insert_history_event({"type": "deploy", "col": "ALCO", "pid": "ALCO_2018__1_2"})

201 insert_history_event({"type": "deploy", "col": "ALCO", "pid": "ALCO_2018__1_3"})

202 insert_history_event({"type": "deploy", "col": "ALCO", "pid": "ALCO_2018__1_4"})

203

204 insert_history_event({"type": "clockss", "pid": "ALL", "col": "ALL"})

205

206

207def insert_history_event(new_event):

208 """

209

210 :param new_event:

211 :return:

212 """

213 if "type" not in new_event or "pid" not in new_event or "col" not in new_event: 213 ↛ 214line 213 didn't jump to line 214 because the condition on line 213 was never true

214 raise ValueError("type, pid or col is nor set")

215

216 do_insert = True

217 last_events = HistoryEvent.objects.all().order_by("-pk")[:1]

218 last_event = last_events[0] if len(last_events) > 0 else None

219

220 if last_event:

221 time_delta = timezone.now() - last_event.created_on

222 # Check if we have to merge with the last event in the history

223 # Match 1 article generates many events per bibitemid

224 if (

225 (

226 last_event.type == new_event["type"]

227 and last_event.type == "matching"

228 and last_event.pid == new_event["pid"]

229 and time_delta < timedelta(seconds=5)

230 )

231 or

232 # Deploy, Archive, Import a collection

233 (

234 last_event.type == new_event["type"]

235 and last_event.pid == new_event["col"]

236 and time_delta < timedelta(seconds=5)

237 )

238 or

239 # Typically used when Edit one article (mark matched ids as valid)

240 (

241 last_event.type == new_event["type"]

242 and last_event.pid == new_event["pid"]

243 and last_event.status == new_event["status"]

244 )

245 ):

246 do_insert = False

247 new_fields = {}

248

249 # Downgrade the status

250 if "status" in new_event and (

251 last_event.status == "OK"

252 or (last_event.status == "WARNING" and new_event["status"] == "ERROR")

253 or new_event["status"] == "ERROR"

254 ):

255 last_event.status = new_event["status"]

256

257 last_event_data = last_event.data

258 new_event_data = new_event["data"] if "data" in new_event else None

259

260 if last_event.pid == new_event["pid"]:

261 # Merge ids

262 if new_event_data and "ids" in new_event["data"]: 262 ↛ 263line 262 didn't jump to line 263 because the condition on line 262 was never true

263 if "ids" in last_event_data:

264 new_fields["ids"] = last_event_data["ids"] + new_event_data["ids"]

265 else:

266 new_fields["ids"] = new_event_data["ids"]

267 elif new_event_data: 267 ↛ 268line 267 didn't jump to line 268 because the condition on line 267 was never true

268 if "articles" in last_event_data and "ids" in new_event_data:

269 # Article inside issue

270 # The last event was for an issue (ex: Matching) and

271 # The new event is for an article.

272 # last_event_data has 'articles' and new_even_data has 'ids'

273 found_articles = [

274 item

275 for item in last_event_data["articles"]

276 if item["pid"] == new_event["pid"]

277 ]

278 if len(found_articles) > 0:

279 article = found_articles[0]

280 article["ids"] = article["ids"] + new_event_data["ids"]

281 else:

282 last_event_data["articles"].append(

283 {"pid": new_event["pid"], "ids": new_event_data["ids"]}

284 )

285 elif "ids" in new_event_data:

286 last_event_data["articles"] = [

287 {"pid": new_event["pid"], "ids": new_event_data["ids"]}

288 ]

289 if "articles" in last_event_data:

290 new_fields["articles"] = last_event_data["articles"]

291

292 # Merge *data['articles']

293 if new_event_data and "articles" in new_event_data: 293 ↛ 294line 293 didn't jump to line 294 because the condition on line 293 was never true

294 if "articles" in last_event_data:

295 articles = last_event_data["articles"]

296 for new_article in new_event_data["articles"]:

297 found_articles = [

298 item for item in articles if item["pid"] == new_article["pid"]

299 ]

300 if len(found_articles) > 0:

301 article = found_articles[0]

302 article["ids"] = article["ids"] + new_article["ids"]

303 else:

304 articles.append(new_article)

305 else:

306 last_event_data["articles"] = new_event_data["articles"]

307 new_fields["articles"] = last_event_data["articles"]

308

309 # Update ids_count

310 if new_event_data and "ids_count" in new_event_data: 310 ↛ 311line 310 didn't jump to line 311 because the condition on line 310 was never true

311 new_count = 0

312 if "ids_count" in last_event_data:

313 new_count = last_event_data["ids_count"]

314 new_count += new_event_data["ids_count"]

315 new_fields["ids_count"] = new_count

316

317 # Deploy, Archive, Import a collection

318 # Add 'issues' to the collection event

319 if last_event.type == new_event["type"] and last_event.pid == new_event["col"]:

320 if "issues" in last_event_data:

321 if new_event["pid"] not in last_event_data["issues"]: 321 ↛ 325line 321 didn't jump to line 325 because the condition on line 321 was always true

322 last_event_data["issues"].append(new_event["pid"])

323 else:

324 last_event_data["issues"] = [new_event["pid"]]

325 new_fields["issues"] = last_event_data["issues"]

326

327 if (

328 new_event_data

329 and "message" in new_event_data

330 and len(new_event_data["message"]) > 0

331 ):

332 if ( 332 ↛ 341line 332 didn't jump to line 341 because the condition on line 332 was always true

333 "message" in last_event_data

334 and new_event["type"] == "edit"

335 and last_event_data["message"] != ""

336 ):

337 new_fields["message"] = (

338 new_event_data["message"] + "<br/>" + last_event_data["message"]

339 )

340 else:

341 new_fields["message"] = new_event_data["message"]

342

343 for key, value in new_fields.items():

344 last_event.data[key] = value

345

346 if "created_on" in new_event:

347 last_event.created_on = new_event["created_on"]

348 else:

349 last_event.created_on = timezone.now()

350

351 last_event.save()

352

353 if do_insert:

354 filtered_events = (

355 HistoryEvent.objects.filter(type=new_event["type"], pid__startswith=new_event["pid"])

356 .exclude(status="OK")

357 .order_by("-pk")

358 )

359

360 first = True

361 for event in filtered_events:

362 if not first or new_event["status"] == "OK": 362 ↛ 365line 362 didn't jump to line 365 because the condition on line 362 was always true

363 event.data["obsolete"] = True

364 event.save()

365 first = False

366

367 event = HistoryEvent(type=new_event["type"], pid=new_event["pid"], col=new_event["col"])

368 if "message" in new_event: 368 ↛ 369line 368 didn't jump to line 369 because the condition on line 368 was never true

369 event.message = new_event["message"]

370 if "created_on" in new_event:

371 event.created_on = new_event["created_on"]

372 if "status" in new_event:

373 event.status = new_event["status"]

374 if "data" in new_event:

375 event.data = new_event["data"]

376

377 event.save()

378

379

380# Attempt to create a generator to group events.

381# See https://docs.python.org/2/library/itertools.html#itertools.groupby

382# It works fine in the model, the view (and the tests)

383# Unfortunately, it does not work with the template.

384# Django's render function converts the iterator into a list:

385# See django/template/defaulttags.py, line 172:

386# if not hasattr(values, '__len__'):

387# values = list(values)

388# len_values = len(values)

389# The generator is completely traversed in list(values) and is not reset

390# When the render arrives in the for loop, there is nothing left to iterate

391class GroupEventsBy:

392 # [k for k, g in groupby('AAAABBBCCDAABBB')] --> A B C D A B

393 # [list(g) for k, g in groupby('AAAABBBCCD')] --> AAAA BBB CC D

394 def __init__(self, iterable, key=None):

395 if key is None:

396

397 def key(x):

398 return x

399

400 self.keyfunc = key

401 self.it = iter(iterable)

402 self.tgtkey = self.currkey = self.currvalue = object()

403 self.error_count = 0

404

405 def __iter__(self):

406 return self

407

408 def __next__(self):

409 while self.currkey == self.tgtkey:

410 self.currvalue = next(self.it) # Exit on StopIteration

411 self.currkey = self.keyfunc(self.currvalue)

412 self.tgtkey = self.currkey

413

414 # make a copy of the iterator to fetch counts

415 self.it, second_it = tee(self.it)

416 value_in_gp = self.currvalue

417 count = error_count = warning_count = 0

418 gp_key = self.tgtkey

419 while self.tgtkey == gp_key:

420 count += 1

421 if value_in_gp.status == "ERROR":

422 error_count += 1

423 if value_in_gp.status == "WARNING":

424 warning_count += 1

425 try:

426 value_in_gp = next(second_it)

427 gp_key = self.keyfunc(value_in_gp)

428 except StopIteration:

429 gp_key = object()

430

431 return (self.currkey, count, error_count, warning_count, self._grouper(self.tgtkey))

432

433 def _grouper(self, tgtkey):

434 while self.currkey == tgtkey:

435 yield self.currvalue

436 self.currvalue = next(self.it) # Exit on StopIteration

437 self.currkey = self.keyfunc(self.currvalue)

438

439

440def get_history_events(filters):

441 data = HistoryEvent.objects.all()

442

443 filter_by_month = True

444 kwargs = {}

445 for key, value in filters.items():

446 if key == "status":

447 if value == "error":

448 kwargs["status"] = "ERROR"

449 else:

450 data = data.filter(Q(status="ERROR") | Q(status="WARNING"))

451 elif key == "month":

452 if value == "all": 452 ↛ 445line 452 didn't jump to line 445 because the condition on line 452 was always true

453 filter_by_month = False

454 else:

455 kwargs[key] = value

456

457 if filter_by_month:

458 today = datetime.datetime.today()

459 kwargs["created_on__year"] = today.year

460 kwargs["created_on__month"] = today.month

461

462 data = data.filter(**kwargs).order_by("-pk").annotate(month=TruncMonth("created_on"))

463 # grouped_events = GroupEventsBy(data, lambda t: t.month)

464 grouped_events = []

465 events_in_month = []

466 curmonth = None

467 error_count = warning_count = 0

468 for event in data:

469 month = event.month

470 if month != curmonth and len(events_in_month) > 0:

471 grouped_events.append(

472 {

473 "month": curmonth,

474 "error_count": error_count,

475 "warning_count": warning_count,

476 "events_in_month": events_in_month,

477 }

478 )

479 events_in_month = []

480 curmonth = month

481 error_count = warning_count = 0

482 elif month != curmonth:

483 curmonth = month

484 events_in_month.append(event)

485 is_obsolete = "obsolete" in event.data and event.data["obsolete"]

486 if event.status == "ERROR" and not is_obsolete:

487 error_count += 1

488 if event.status == "WARNING" and not is_obsolete: 488 ↛ 489line 488 didn't jump to line 489 because the condition on line 488 was never true

489 warning_count += 1

490 if len(events_in_month) > 0: 490 ↛ 500line 490 didn't jump to line 500 because the condition on line 490 was always true

491 grouped_events.append(

492 {

493 "month": curmonth,

494 "error_count": error_count,

495 "warning_count": warning_count,

496 "events_in_month": events_in_month,

497 }

498 )

499

500 return grouped_events

501

502 # .values('month') #.annotate(count=Count('id'))

503

504 # https://stackoverflow.com/questions/8746014/django-group-by-date-day-month-year

505 # data = test1.objects.annotate(month=TruncMonth('cdate')).values('month').annotate(c=Count('id')).order_by()

506

507 # https://docs.djangoproject.com/en/2.1/topics/db/aggregation/#values

508

509

510def get_history_error_warning_counts():

511 # .exclude(data__obsolete=True) or ~Q(data__obsolete=True) do not work in Django 1.11

512 # Need to count manually

513

514 error_count = warning_count = 0

515

516 events = HistoryEvent.objects.filter(status="ERROR")

517 for event in events:

518 if "obsolete" not in event.data or not event.data["obsolete"]:

519 error_count += 1

520

521 events = HistoryEvent.objects.filter(status="WARNING")

522 for event in events:

523 if "obsolete" not in event.data or not event.data["obsolete"]:

524 warning_count += 1

525

526 return error_count, warning_count

527

528

529def delete_history_event(pk):

530 HistoryEvent.objects.get(pk=pk).delete()

531

532

533def get_history_last_event_by(type, pid=""):

534 last_events = HistoryEvent.objects.filter(type=type, status="OK")

535 if len(pid) > 0:

536 last_events = last_events.filter(pid__startswith=pid)

537 last_events = last_events.order_by("-pk")[:1]

538

539 last_event = last_events[0] if len(last_events) > 0 else None

540 return last_event

541

542

543def get_gap(now, event):

544 gap = ""

545 if event:

546 timelapse = now - event.created_on

547 if timelapse.days > 0:

548 gap = str(timelapse.days) + " days ago"

549 elif timelapse.seconds > 3600:

550 gap = str(int(timelapse.seconds // 3600)) + " hours ago"

551 else:

552 gap = str(int(timelapse.seconds // 60)) + " minutes ago"

553 return gap