Coverage for manila/data/utils.py: 98%

1# Copyright 2015, Hitachi Data Systems. 

2# 

3# Licensed under the Apache License, Version 2.0 (the "License"); you may 

4# not use this file except in compliance with the License. You may obtain 

5# a copy of the License at 

6# 

7# http://www.apache.org/licenses/LICENSE-2.0 

8# 

9# Unless required by applicable law or agreed to in writing, software 

10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 

11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 

12# License for the specific language governing permissions and limitations 

13# under the License. 

14 

15import os 

16 

17from oslo_log import log 

18 

19from manila import exception 

20from manila.i18n import _ 

21from manila import utils 

22 

23LOG = log.getLogger(__name__) 

24 

25 

26class Copy(object): 

27 

28 def __init__(self, src, dest, ignore_list, check_hash=False): 

29 self.src = src 

30 self.dest = dest 

31 self.total_size = 0 

32 self.current_size = 0 

33 self.files = [] 

34 self.dirs = [] 

35 self.current_copy = None 

36 self.ignore_list = ignore_list 

37 self.cancelled = False 

38 self.initialized = False 

39 self.completed = False 

40 self.check_hash = check_hash 

41 

42 def get_progress(self): 

43 

44 # Empty share or empty contents 

45 if self.completed and self.total_size == 0: 

46 return {'total_progress': 100} 

47 

48 if not self.initialized or self.current_copy is None: 

49 return {'total_progress': 0} 

50 

51 try: 

52 size, err = utils.execute("stat", "-c", "%s", 

53 self.current_copy['file_path'], 

54 run_as_root=True) 

55 size = int(size) 

56 except utils.processutils.ProcessExecutionError: 

57 size = 0 

58 

59 current_file_progress = 0 

60 if self.current_copy['size'] > 0: 60 ↛ 62line 60 didn't jump to line 62 because the condition on line 60 was always true

61 current_file_progress = size * 100 / self.current_copy['size'] 

62 current_file_path = self.current_copy['file_path'] 

63 

64 total_progress = 0 

65 if self.total_size > 0: 65 ↛ 71line 65 didn't jump to line 71 because the condition on line 65 was always true

66 if current_file_progress == 100: 

67 size = 0 

68 total_progress = int((self.current_size + size) * 

69 100 / self.total_size) 

70 

71 progress = { 

72 'total_progress': total_progress, 

73 'current_file_path': current_file_path, 

74 'current_file_progress': current_file_progress 

75 } 

76 

77 return progress 

78 

79 def cancel(self): 

80 

81 self.cancelled = True 

82 

83 def run(self): 

84 

85 self.get_total_size(self.src) 

86 self.initialized = True 

87 self.copy_data(self.src) 

88 self.copy_stats(self.src) 

89 self.completed = True 

90 

91 LOG.info(self.get_progress()) 

92 

93 def get_total_size(self, path): 

94 if self.cancelled: 

95 return 

96 out, err = utils.execute( 

97 "ls", "-pA1", "--group-directories-first", path, 

98 run_as_root=True) 

99 for line in out.split('\n'): 

100 if self.cancelled: 

101 return 

102 if len(line) == 0: 

103 continue 

104 src_item = os.path.join(path, line) 

105 if line[-1] == '/': 

106 if line[0:-1] in self.ignore_list: 

107 continue 

108 self.get_total_size(src_item) 

109 else: 

110 if line in self.ignore_list: 

111 continue 

112 size, err = utils.execute("stat", "-c", "%s", src_item, 

113 run_as_root=True) 

114 self.total_size += int(size) 

115 

116 def copy_data(self, path): 

117 if self.cancelled: 

118 return 

119 out, err = utils.execute( 

120 "ls", "-pA1", "--group-directories-first", path, 

121 run_as_root=True) 

122 for line in out.split('\n'): 

123 if self.cancelled: 

124 return 

125 if len(line) == 0: 

126 continue 

127 src_item = os.path.join(path, line) 

128 dest_item = src_item.replace(self.src, self.dest) 

129 if line[-1] == '/': 

130 if line[0:-1] in self.ignore_list: 

131 continue 

132 utils.execute("mkdir", "-p", dest_item, run_as_root=True) 

133 self.copy_data(src_item) 

134 else: 

135 if line in self.ignore_list: 

136 continue 

137 size, err = utils.execute("stat", "-c", "%s", src_item, 

138 run_as_root=True) 

139 

140 self.current_copy = {'file_path': dest_item, 

141 'size': int(size)} 

142 

143 self._copy_and_validate(src_item, dest_item) 

144 

145 self.current_size += int(size) 

146 LOG.info(self.get_progress()) 

147 

148 @utils.retry(retry_param=exception.ShareDataCopyFailed, retries=2) 

149 def _copy_and_validate(self, src_item, dest_item): 

150 utils.execute("cp", "-P", "--preserve=all", src_item, 

151 dest_item, run_as_root=True) 

152 

153 if self.check_hash: 153 ↛ exitline 153 didn't return from function '_copy_and_validate' because the condition on line 153 was always true

154 _validate_item(src_item, dest_item) 

155 

156 def copy_stats(self, path): 

157 if self.cancelled: 

158 return 

159 out, err = utils.execute( 

160 "ls", "-pA1", "--group-directories-first", path, 

161 run_as_root=True) 

162 for line in out.split('\n'): 

163 if self.cancelled: 

164 return 

165 if len(line) == 0: 

166 continue 

167 src_item = os.path.join(path, line) 

168 dest_item = src_item.replace(self.src, self.dest) 

169 # NOTE(ganso): Should re-apply attributes for folders. 

170 if line[-1] == '/': 

171 if line[0:-1] in self.ignore_list: 

172 continue 

173 self.copy_stats(src_item) 

174 utils.execute("chmod", "--reference=%s" % src_item, dest_item, 

175 run_as_root=True) 

176 utils.execute("touch", "--reference=%s" % src_item, dest_item, 

177 run_as_root=True) 

178 utils.execute("chown", "--reference=%s" % src_item, dest_item, 

179 run_as_root=True) 

180 

181 

182def _validate_item(src_item, dest_item): 

183 src_sum, err = utils.execute( 

184 "sha256sum", "%s" % src_item, run_as_root=True) 

185 dest_sum, err = utils.execute( 

186 "sha256sum", "%s" % dest_item, run_as_root=True) 

187 if src_sum.split()[0] != dest_sum.split()[0]: 187 ↛ exitline 187 didn't return from function '_validate_item' because the condition on line 187 was always true

188 msg = _("Data corrupted while copying. Aborting data copy.") 

189 raise exception.ShareDataCopyFailed(reason=msg)