Coverage for src/ptf_tools/tasks.py: 22%

1import os 

2import subprocess 

3 

4from celery import shared_task 

5from django.conf import settings 

6from ptf.cmds.ptf_cmds import ( 

7 archiveCollectionPtfCmd, 

8 archiveIssuePtfCmd, 

9 archiveNumdamResourcePtfCmd, 

10 get_numdam_issues_list, 

11) 

12from ptf.models import Article, Collection, Container 

13 

14from history.views import insert_history_event, manage_exceptions 

15 

16 

17@shared_task 

18def archive_numdam_collection(colid): 

19 """ 

20 Archive the files related to a collection (top level only, does not archive files of the issues) 

21 => col.xml and the collection images 

22 """ 

23 

24 if colid in settings.MERSENNE_COLLECTIONS: 

25 return 

26 

27 try: 

28 # First, check NFS mount. A Bad mount will result in a timeout with os.path.isfile or isdir 

29 # The code will hang and the Celery tasks will be killed at some point 

30 # It's better to check and raise an exception 

31 subprocess.check_call(["test", "-d", settings.NUMDAM_ISSUE_SRC_FOLDER], timeout=0.5) 

32 subprocess.check_call(["test", "-d", settings.NUMDAM_ARTICLE_SRC_FOLDER], timeout=0.5) 

33 subprocess.check_call( 

34 ["test", "-d", os.path.join(settings.NUMDAM_DATA_ROOT, colid)], timeout=0.5 

35 ) 

36 

37 archiveNumdamResourcePtfCmd({"colid": colid}).do() 

38 

39 pids = sorted(get_numdam_issues_list(colid)) 

40 for pid in pids: 

41 print("1 task (issue)") 

42 archive_numdam_issue.delay(colid, pid) 

43 

44 # except requests.exceptions.RequestException as exception: 

45 # manage_exceptions("archive", colid, colid, "WARNING", exception) 

46 # raise 

47 except Exception as exception: 

48 manage_exceptions("archive", colid, colid, "ERROR", exception) 

49 raise 

50 

51 insert_history_event( 

52 {"type": "archive", "pid": colid, "col": colid, "status": "OK", "data": {"message": ""}} 

53 ) 

54 

55 

56@shared_task 

57def archive_numdam_issue(colid, pid): 

58 """ 

59 Archive the files of an issue. Get the list of files from numdam.org 

60 """ 

61 try: 

62 archiveNumdamResourcePtfCmd({"colid": colid, "pid": pid}).do() 

63 # except requests.exceptions.RequestException as exception: 

64 # manage_exceptions("archive", pid, colid, "WARNING", exception) 

65 # raise 

66 except Exception as exception: 

67 manage_exceptions("archive", pid, colid, "ERROR", exception) 

68 raise 

69 

70 

71@shared_task 

72def archive_trammel_collection(colid, mathdoc_archive, binary_files_folder): 

73 try: 

74 # First, check NFS mount. A Bad mount will result in a timeout with os.path.isfile or isdir 

75 # The code will hang and the Celery tasks will be killed at some point 

76 # It's better to check and raise an exception 

77 subprocess.check_call(["test", "-d", settings.NUMDAM_ISSUE_SRC_FOLDER], timeout=0.5) 

78 subprocess.check_call(["test", "-d", settings.NUMDAM_ARTICLE_SRC_FOLDER], timeout=0.5) 

79 subprocess.check_call(["test", "-d", settings.CEDRAM_TEX_FOLDER], timeout=0.5) 

80 

81 collection = Collection.objects.get(pid=colid) 

82 issues = collection.content.all() 

83 

84 archiveCollectionPtfCmd({"colid": colid, "issues": issues}).do() 

85 for issue in issues: 

86 archive_trammel_resource.delay(colid, issue.pid, mathdoc_archive, binary_files_folder) 

87 

88 except Exception as exception: 

89 manage_exceptions("archive", colid, colid, "ERROR", exception) 

90 raise 

91 

92 insert_history_event( 

93 {"type": "archive", "pid": colid, "col": colid, "status": "OK", "data": {"message": ""}} 

94 ) 

95 

96 

97@shared_task 

98def archive_trammel_resource( 

99 colid=None, pid=None, mathdoc_archive=None, binary_files_folder=None, article_doi=None 

100): 

101 """ 

102 Archive the files of an issue or an article stored in the ptf-tools database (Trammel) 

103 """ 

104 try: 

105 if article_doi is not None: 

106 article = Article.objects.get(doi=article_doi) 

107 cmd = archiveIssuePtfCmd( 

108 { 

109 "pid": pid, 

110 "export_folder": mathdoc_archive, 

111 "binary_files_folder": binary_files_folder, 

112 "article": article, 

113 } 

114 ) 

115 else: 

116 issue = Container.objects.get(pid=pid) 

117 cmd = archiveIssuePtfCmd( 

118 { 

119 "pid": issue.pid, 

120 "export_folder": mathdoc_archive, 

121 "binary_files_folder": binary_files_folder, 

122 } 

123 ) 

124 cmd.do() 

125 except Exception as exception: 

126 manage_exceptions("archive", pid, colid, "ERROR", exception) 

127 raise 

128 

129 insert_history_event( 

130 {"type": "archive", "pid": pid, "col": colid, "status": "OK", "data": {"message": ""}} 

131 )