Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/bin/env python3 

2# 

3# Copyright (C) 2024 Vates SAS 

4# 

5# This program is free software: you can redistribute it and/or modify 

6# it under the terms of the GNU General Public License as published by 

7# the Free Software Foundation, either version 3 of the License, or 

8# (at your option) any later version. 

9# This program is distributed in the hope that it will be useful, 

10# but WITHOUT ANY WARRANTY; without even the implied warranty of 

11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

12# GNU General Public License for more details. 

13# 

14# You should have received a copy of the GNU General Public License 

15# along with this program. If not, see <https://www.gnu.org/licenses/>. 

16 

17from sm_typing import Any, Callable, Dict, Final, Optional, Tuple, cast, override 

18from typing import BinaryIO 

19 

20import errno 

21import os 

22import re 

23import time 

24import struct 

25import zlib 

26import json 

27from pathlib import Path 

28from enum import IntFlag 

29 

30import util 

31import xs_errors 

32from blktap2 import TapCtl 

33from cowutil import CowUtil, CowImageInfo 

34from lvmcache import LVMCache 

35from constants import NS_PREFIX_LVM, VG_PREFIX 

36 

37# ------------------------------------------------------------------------------ 

38 

39MAX_QCOW_CHAIN_LENGTH: Final = 30 

40MAX_UNDEFINED_QUERY_ATTEMPTS: Final = 10 # Max number of undefined accepted during an onlineCoalesce before raising an error 

41 

42QCOW2_DEFAULT_CLUSTER_SIZE: Final = 64 * 1024 # 64 KiB 

43 

44MIN_QCOW_SIZE: Final = QCOW2_DEFAULT_CLUSTER_SIZE 

45 

46MAX_QCOW_SIZE: Final = 17589500641280 # Max size so that the fully allocated size with metadata is under the max size of EXT4 (17592185061376 bytes fully allocated) 

47 

48QEMU_IMG: Final = "/usr/bin/qemu-img" 

49QCOW2_HELPER = "/opt/xensource/libexec/qcow2_helper" 

50 

51QCOW2_TYPE: Final = "qcow2" 

52RAW_TYPE: Final = "raw" 

53 

54# ------------------------------------------------------------------------------ 

55 

56class QCowUtil(CowUtil): 

57 

58 # We followed specifications found here: 

59 # https://github.com/qemu/qemu/blob/master/docs/interop/qcow2.txt 

60 

61 QCOW2_MAGIC = 0x514649FB # b"QFI\xfb": Magic number for QCOW2 files 

62 QCOW2_HEADER_SIZE = 104 # In fact the last information we need is at offset 40-47 

63 QCOW2_BACKING_FILE_OFFSET = 8 

64 QCOW2_INCOMPATIBLE_FEATURE_MASK = 0x0000_0000_0000_001F # Bit 0-4 

65 class IncompatibleFeatures(IntFlag): 

66 DIRTY = 0x01 

67 CORRUPT = 0x02 

68 EXTERNAL_DATA_FILE = 0x04 

69 COMPRESSION_TYPE = 0x08 

70 EXTENDED_L2 = 0x10 

71 

72 def _read_qcow2(self, path: str): 

73 with open(path, "rb") as qcow2_file: 

74 self.filename = path # Keep the filename if clean is called 

75 self.header = self._read_qcow2_header(qcow2_file) 

76 

77 @staticmethod 

78 def _read_qcow2_backingfile(file: BinaryIO, backing_file_offset: int , backing_file_size: int) -> str: 

79 if backing_file_offset == 0: 

80 return "" 

81 

82 file.seek(backing_file_offset) 

83 parent_name = file.read(backing_file_size) 

84 return parent_name.decode("UTF-8") 

85 

86 @staticmethod 

87 def _read_qcow2_header(file: BinaryIO) -> Dict[str, Any]: 

88 """Returns a dict containing some information from QCow2 header. 

89 

90 Args: 

91 file: The qcow2 file object. 

92 

93 Returns: 

94 dict: magic, version, cluster_bits, l1_size and l1_table_offset. 

95 

96 Raises: 

97 ValueError: if qcow2 magic is not recognized or cluster size not supported. 

98 """ 

99 # The header is as follow: 

100 # 

101 # magic: u32, // Magic string "QFI\xfb" 

102 # version: u32, // Version (2 or 3) 

103 # backing_file_offset: u64, // Offset to the backing file name 

104 # backing_file_size: u32, // Size of the backing file name 

105 # cluster_bits: u32, // Bits used for addressing within a cluster 

106 # size: u64, // Virtual disk size 

107 # crypt_method: u32, // 0 = no encryption, 1 = AES encryption 

108 # l1_size: u32, // Number of entries in the L1 table 

109 # l1_table_offset: u64, // Offset to the active L1 table 

110 # refcount_table_offset: u64, // Offset to the refcount table 

111 # refcount_table_clusters: u32, // Number of clusters for the refcount table 

112 # nb_snapshots: u32, // Number of snapshots in the image 

113 # snapshots_offset: u64, // Offset to the snapshot table 

114 # incompatible_features: u64, // Bitmask of incompatible features 

115 

116 file.seek(0) 

117 header = file.read(QCowUtil.QCOW2_HEADER_SIZE) 

118 ( 

119 magic, 

120 version, 

121 backing_file_offset, 

122 backing_file_size, 

123 cluster_bits, 

124 size, 

125 _, 

126 l1_size, 

127 l1_table_offset, 

128 refcount_table_offset, 

129 _, 

130 _, 

131 snapshots_offset, 

132 incompatible_features, 

133 ) = struct.unpack(">IIQIIQIIQQIIQQ", header[:80]) 

134 

135 if magic != QCowUtil.QCOW2_MAGIC: 

136 raise ValueError("Not a valid QCOW2 file") 

137 

138 if incompatible_features & ~QCowUtil.QCOW2_INCOMPATIBLE_FEATURE_MASK: 

139 raise ValueError("QCOW2: Unknown incompatible feature bit set") 

140 

141 if incompatible_features & QCowUtil.IncompatibleFeatures.DIRTY: 

142 raise ValueError("QCOW2: Image is dirty") 

143 

144 if incompatible_features & QCowUtil.IncompatibleFeatures.CORRUPT: 

145 raise ValueError("QCOW2: Image is corrupt; cannot be opened read/write") 

146 

147 if incompatible_features & QCowUtil.IncompatibleFeatures.EXTERNAL_DATA_FILE: 

148 raise ValueError("QCOW2: Image with external data file is unsupported") 

149 

150 if incompatible_features & QCowUtil.IncompatibleFeatures.COMPRESSION_TYPE: 

151 raise ValueError("QCOW2: Image compression is not supported") 

152 

153 parent_name = QCowUtil._read_qcow2_backingfile(file, backing_file_offset, backing_file_size) 

154 

155 return { 

156 "version": version, 

157 "backing_file_offset": backing_file_offset, 

158 "backing_file_size": backing_file_size, 

159 "virtual_disk_size": size, 

160 "cluster_bits": cluster_bits, 

161 "l1_size": l1_size, 

162 "l1_table_offset": l1_table_offset, 

163 "refcount_table_offset": refcount_table_offset, 

164 "snapshots_offset": snapshots_offset, 

165 "parent": parent_name, 

166 "incompatible_features": incompatible_features, 

167 } 

168 

169 @staticmethod 

170 def _move_backing_file( 

171 f: BinaryIO, old_offset: int, new_offset: int, data_size: int 

172 ) -> None: 

173 """Move a number of bytes from old_offset to new_offset and replaces the old 

174 value by 0s. It is up to the caller to save the current position in the file 

175 if needed. 

176 

177 Args: 

178 f: the file the will be modified 

179 old_offset: the current offset 

180 new_offset: the new offset where we want to move data 

181 data_size: Size in bytes of data that we want to move 

182 

183 Returns: 

184 Nothing but the file f is modified and the position in the file also. 

185 """ 

186 # Read the string at backing_file_offset 

187 f.seek(old_offset) 

188 data = f.read(data_size) 

189 

190 # Write zeros at the original location 

191 f.seek(old_offset) 

192 f.write(b"\x00" * data_size) 

193 

194 # Write the string to the new location 

195 f.seek(new_offset) 

196 f.write(data) 

197 

198 def _add_or_find_custom_header(self) -> int: 

199 """Add custom header at the end of header extensions 

200 

201 It finds the end of the header extensions and add the custom header. 

202 If the header already exists nothing is done. 

203 

204 Args: 

205 

206 Returns: 

207 It returns the data offset where custom header is found or created. 

208 If data offset is 0 something weird happens. 

209 The qcow2 file in self.filename can be modified. 

210 """ 

211 header_length = 72 # This is the default value for version 2 images 

212 

213 custom_header_type = 0x76617465 # vate: it is easy to recognize with hexdump -C 

214 custom_header_length = 8 

215 custom_header_data = 0 

216 # We don't need padding because we are already aligned 

217 custom_header = struct.pack( 

218 ">IIQ", custom_header_type, custom_header_length, custom_header_data 

219 ) 

220 

221 with open(self.filename, "rb+") as qcow2_file: 

222 if self.header["version"] == 3: 

223 qcow2_file.seek(100) # 100 is the offset of header_length 

224 header_length = int.from_bytes(qcow2_file.read(4), "big") 

225 

226 # After the image header we found Header extension. So we need to find the end of 

227 # the header extension area and add our custom header. 

228 qcow2_file.seek(header_length) 

229 

230 custom_data_offset = 0 

231 

232 while True: 

233 ext_type = int.from_bytes(qcow2_file.read(4), "big") 

234 ext_len = int.from_bytes(qcow2_file.read(4), "big") 

235 

236 if ext_type == custom_header_type: 

237 # A custom header is already there 

238 custom_data_offset = qcow2_file.tell() 

239 break 

240 

241 if ext_type == 0x00000000: 

242 # End mark found. If we found the end mark it means that we didn't find 

243 # the custom header. So we need to add it. 

244 custom_data_offset = qcow2_file.tell() 

245 

246 # We will overwrite the end marker so rewind a little bit to 

247 # write the new type extension and the new length. But if there is 

248 # a backing file we need to move it to make some space. 

249 if self.header["backing_file_offset"]: 

250 # Keep current position 

251 saved_pos = qcow2_file.tell() 

252 

253 bf_offset = self.header["backing_file_offset"] 

254 bf_size = self.header["backing_file_size"] 

255 bf_new_offset = bf_offset + len(custom_header) 

256 self._move_backing_file( 

257 qcow2_file, bf_offset, bf_new_offset, bf_size 

258 ) 

259 

260 # Update the header to match the new backing file offset 

261 self.header["backing_file_offset"] = bf_new_offset 

262 qcow2_file.seek(self.QCOW2_BACKING_FILE_OFFSET) 

263 qcow2_file.write(struct.pack(">Q", bf_new_offset)) 

264 

265 # Restore saved position 

266 qcow2_file.seek(saved_pos) 

267 

268 qcow2_file.seek(-8, 1) 

269 qcow2_file.write(custom_header) 

270 break 

271 

272 # Round up the header extension size to the next multiple of 8 

273 ext_len = (ext_len + 7) & 0xFFFFFFF8 

274 qcow2_file.seek(ext_len, 1) 

275 

276 return custom_data_offset 

277 

278 def _has_extended_l2(self, path: str) -> bool: 

279 self._read_qcow2(path) 

280 return bool(self.header["incompatible_features"] & self.IncompatibleFeatures.EXTENDED_L2) 

281 

282 @staticmethod 

283 def _convert_bool_to_qemu_option_value(option: bool) -> str: 

284 return 'on' if option else 'off' 

285 

286 # ---- 

287 # Implementation of CowUtil 

288 # ---- 

289 

290 @override 

291 def getMinImageSize(self) -> int: 

292 return MIN_QCOW_SIZE 

293 

294 @override 

295 def getMaxImageSize(self) -> int: 

296 return MAX_QCOW_SIZE 

297 

298 @override 

299 def getBlockSize(self, path: str) -> int: 

300 self._read_qcow2(path) 

301 return 1 << self.header["cluster_bits"] 

302 

303 @override 

304 def getFooterSize(self) -> int: 

305 return 0 

306 

307 @override 

308 def getDefaultPreallocationSizeVirt(self) -> int: 

309 """vhdutil answer max size (2TiB) here but we don't want to allocate for max size in QCOW2, it would make small LV a lot bigger.""" 

310 return MIN_QCOW_SIZE 

311 

312 @override 

313 def getMaxChainLength(self) -> int: 

314 return MAX_QCOW_CHAIN_LENGTH 

315 

316 @override 

317 def calcOverheadEmpty(self, virtual_size: int, block_size: Optional[int] = None) -> int: 

318 if block_size: 

319 cluster_size = block_size 

320 else: 

321 cluster_size = QCOW2_DEFAULT_CLUSTER_SIZE 

322 cmd = [QEMU_IMG, "measure", "-O", "qcow2", "--output", "json", "-o", f"cluster_size={cluster_size},extended_l2=on", "--size", f"{virtual_size}"] 

323 output = json.loads(self._ioretry(cmd)) 

324 return int(output["required"]) 

325 

326 @override 

327 def calcOverheadBitmap(self, virtual_size: int) -> int: 

328 """ 

329 We are not impacted, this is only for VHD. We send back 0 for QCOW2. 

330 """ 

331 return 0 

332 

333 @override 

334 def getInfo( 

335 self, 

336 path: str, 

337 extractUuidFunction: Callable[[str], str], 

338 includeParent: bool = True, 

339 resolveParent: bool = True, 

340 useBackupFooter: bool = False 

341 ) -> CowImageInfo: 

342 """ 

343 resolveParent is useful to find the absolute path of a parent since the path stored in VHD is relative. 

344 For linstor, this cause issues since the parent might not exist locally. 

345 We currently do not implement this for QCOW2. 

346 """ 

347 self._read_qcow2(path) 

348 uuid = extractUuidFunction(path) 

349 cowinfo = CowImageInfo(uuid) 

350 cowinfo.path = path 

351 cowinfo.sizeVirt = self.header["virtual_disk_size"] 

352 cowinfo.sizePhys = self.getSizePhys(path) 

353 cowinfo.hidden = self.getHidden(path) 

354 if includeParent: 

355 parent_path = self.header["parent"] 

356 if parent_path != "": 

357 cowinfo.parentPath = parent_path 

358 cowinfo.parentUuid = extractUuidFunction(parent_path) 

359 

360 return cowinfo 

361 

362 @override 

363 def getInfoFromLVM( 

364 self, lvName: str, extractUuidFunction: Callable[[str], str], vgName: str 

365 ) -> Optional[CowImageInfo]: 

366 lvcache = LVMCache(vgName) 

367 return self._getInfoLV(lvcache, extractUuidFunction, vgName, lvName) 

368 

369 def _getInfoLV( 

370 self, lvcache: LVMCache, extractUuidFunction: Callable[[str], str], vgName: str, lvName: str 

371 ) -> Optional[CowImageInfo]: 

372 lvPath = "/dev/{}/{}".format(vgName, lvName) 

373 lvcache.refresh() 

374 if lvName not in lvcache.lvs: 

375 util.SMlog("{} does not exist anymore".format(lvName)) 

376 return None 

377 

378 vdiUuid = extractUuidFunction(lvPath) 

379 srUuid = vgName.replace(VG_PREFIX, "") 

380 

381 ns = NS_PREFIX_LVM + srUuid 

382 lvcache.activate(ns, vdiUuid, lvName, False) 

383 try: 

384 cowinfo = self.getInfo(lvPath, extractUuidFunction) 

385 finally: 

386 lvcache.deactivate(ns, vdiUuid, lvName, False) 

387 return cowinfo 

388 

389 @override 

390 def getAllInfoFromVG( 

391 self, 

392 pattern: str, 

393 extractUuidFunction: Callable[[str], str], 

394 vgName: Optional[str] = None, 

395 parents: bool = False, 

396 exitOnError: bool = False 

397 ) -> Dict[str, CowImageInfo]: 

398 result: Dict[str, CowImageInfo] = dict() 

399 #TODO: handle exitOnError 

400 if vgName: 400 ↛ 401line 400 didn't jump to line 401, because the condition on line 400 was never true

401 reg = re.compile(pattern) 

402 lvcache = LVMCache(vgName) 

403 lvcache.refresh() 

404 # We get size in lvcache.lvs[lvName].size (in bytes) 

405 # We could read the header from the PV directly 

406 lvList = list(lvcache.lvs.keys()) 

407 for lvName in lvList: 

408 # lvinfo = lvcache.lvs[lvName] 

409 if reg.match(lvName): 

410 cowinfo = self._getInfoLV(lvcache, extractUuidFunction, vgName, lvName) 

411 if cowinfo is None: #We get None if the LV stopped existing in the meanwhile 

412 continue 

413 cowinfo.path = lvName # Function CowUtil.getParentChain expect lvName here, otherwise blktap.{_activate,_deactivate} crashes 

414 result[cowinfo.uuid] = cowinfo 

415 if parents: 

416 parentUuid = cowinfo.parentUuid 

417 parentPath = cowinfo.parentPath 

418 while parentUuid != "": 

419 parentLvName = parentPath.split("/")[-1] 

420 parent_cowinfo = self._getInfoLV(lvcache, extractUuidFunction, vgName, parentLvName) 

421 if parent_cowinfo is None: #Parent disappeared while scanning 

422 raise util.SMException("Parent of {} wasn't found during scan".format(lvName)) 

423 parentUuid = parent_cowinfo.parentUuid 

424 parentPath = parent_cowinfo.parentPath 

425 parent_cowinfo.path = parentLvName #Same reason as above, some users expect LvName here instead of path 

426 result[parent_cowinfo.uuid] = parent_cowinfo 

427 

428 return result 

429 else: 

430 pattern_p: Path = Path(pattern) 

431 list_qcow = list(pattern_p.parent.glob(pattern_p.name)) 

432 for qcow in list_qcow: 432 ↛ 433line 432 didn't jump to line 433, because the loop on line 432 never started

433 qcow_str = str(qcow) 

434 info = self.getInfo(qcow_str, extractUuidFunction) 

435 result[info.uuid] = info 

436 return result 

437 

438 @override 

439 def getParent(self, path: str, extractUuidFunction: Callable[[str], str]) -> Optional[str]: 

440 parent = self.getParentNoCheck(path) 

441 if parent: 

442 return extractUuidFunction(parent) 

443 return None 

444 

445 @override 

446 def getParentNoCheck(self, path: str) -> Optional[str]: 

447 self._read_qcow2(path) 

448 parent_path = self.header["parent"] 

449 if parent_path == "": 

450 return None 

451 return parent_path 

452 

453 @override 

454 def hasParent(self, path: str) -> bool: 

455 if self.getParentNoCheck(path): 

456 return True 

457 return False 

458 

459 @override 

460 def setParent(self, path: str, parentPath: str, parentRaw: bool) -> None: 

461 pid_openers = util.get_openers_pid(path) 

462 if pid_openers: 

463 util.SMlog("Rebasing while process {} has the VDI opened".format(pid_openers)) 

464 

465 parentType = QCOW2_TYPE 

466 if parentRaw: 

467 parentType = RAW_TYPE 

468 cmd = [QEMU_IMG, "rebase", "-q", "-u", "-f", QCOW2_TYPE, "-F", parentType, "-b", parentPath, path] 

469 self._ioretry(cmd) 

470 

471 @override 

472 def getHidden(self, path: str) -> bool: 

473 """Get hidden property according to the value b 

474 

475 Args: 

476 

477 Returns: 

478 True if hidden is set, False otherwise 

479 """ 

480 self._read_qcow2(path) 

481 custom_data_offset = self._add_or_find_custom_header() 

482 if custom_data_offset == 0: 

483 raise util.SMException("Custom data offset not found... should not reach this") 

484 

485 with open(path, "rb") as qcow2_file: 

486 qcow2_file.seek(custom_data_offset) 

487 hidden = qcow2_file.read(1) 

488 if hidden == b"\x00": 

489 return False 

490 return True 

491 

492 @override 

493 def setHidden(self, path: str, hidden: bool = True) -> None: 

494 """Set hidden property according to the value b 

495 

496 Args: 

497 bool: True if you want to set the property. False otherwise 

498 

499 Returns: 

500 nothing. If the custom headers is not found it is created so the 

501 qcow file can be modified. 

502 """ 

503 self._read_qcow2(path) 

504 custom_data_offset = self._add_or_find_custom_header() 

505 if custom_data_offset == 0: 

506 raise util.SMException("Custom data offset not found... should not reach this") 

507 

508 with open(self.filename, "rb+") as qcow2_file: 

509 qcow2_file.seek(custom_data_offset) 

510 if hidden: 

511 qcow2_file.write(b"\x01") 

512 else: 

513 qcow2_file.write(b"\x00") 

514 

515 @override 

516 def getSizeVirt(self, path: str) -> int: 

517 self._read_qcow2(path) 

518 return self.header['virtual_disk_size'] 

519 

520 @override 

521 def setSizeVirt(self, path: str, size: int, jFile: str) -> None: 

522 """ 

523 size: byte 

524 jFile: a journal file used for resizing with VHD, not useful for QCOW2 

525 """ 

526 cmd = [QEMU_IMG, "resize", "-q", path, str(size)] 

527 self._ioretry(cmd) 

528 

529 @override 

530 def setSizeVirtFast(self, path: str, size: int) -> None: 

531 self.setSizeVirt(path, size, "") 

532 

533 @override 

534 def getMaxResizeSize(self, path: str) -> int: 

535 return 0 

536 

537 @override 

538 def getSizePhys(self, path: str) -> int: 

539 size = os.stat(path).st_size 

540 if size == 0: 

541 size = int(self._ioretry(["blockdev", "--getsize64", path])) 

542 return size 

543 

544 @override 

545 def setSizePhys(self, path: str, size: int, debug: bool = True) -> None: 

546 pass #We do not store the physical size in the image in QCOW2 

547 

548 @override 

549 def getAllocatedSize(self, path: str) -> int: 

550 cmd = [QCOW2_HELPER, "allocated", path] 

551 return int(self._ioretry(cmd)) 

552 

553 @override 

554 def getResizeJournalSize(self) -> int: 

555 return 0 

556 

557 @override 

558 def killData(self, path: str) -> None: 

559 """Remove all data and reset L1/L2 table. 

560 

561 Args: 

562 self: The QcowInfo object. 

563 

564 Returns: 

565 nothing. 

566 """ 

567 self._read_qcow2(path) 

568 # We need to reset L1 entries and then just truncate the file right 

569 # after L1 entries 

570 with open(self.filename, "r+b") as file: 

571 l1_table_offset = self.header["l1_table_offset"] 

572 file.seek(l1_table_offset) 

573 

574 l1_table_size = ( 

575 self.header["l1_size"] * 8 

576 ) # size in bytes, each entry is 8 bytes 

577 file.write(b"\x00" * l1_table_size) 

578 file.truncate(l1_table_offset + l1_table_size) 

579 

580 @override 

581 def getDepth(self, path: str) -> int: 

582 cmd = [QEMU_IMG, "info", "--backing-chain", "--output=json", path] 

583 ret = str(self._ioretry(cmd)) 

584 depth = len(re.findall("\"backing-filename\"", ret))+1 

585 #chain depth is beginning at one for VHD, meaning a VHD without parent has depth = 1 

586 return depth 

587 

588 @override 

589 def getBlockBitmap(self, path: str) -> bytes: 

590 cmd = [QCOW2_HELPER, "bitmap", path] 

591 text = cast(bytes, self._ioretry(cmd, text=False)) 

592 return zlib.compress(text) 

593 

594 def _getTapdisk(self, path: str) -> Tuple[int, int]: 

595 """ 

596 Return a tuple of (PID, Minor) for the given path 

597 """ 

598 pid_openers = util.get_openers_pid(path) 

599 if pid_openers: 

600 if len(pid_openers) > 1: 

601 raise xs_errors.XenError("Multiple openers for {}".format(path)) # TODO: There might be multiple PID? Yes, we can have the chain enabled for multiple leaf (i.e. after a clone), taken into account in cleanup.py 

602 pid = pid_openers[0] 

603 tapdiskList = TapCtl.list(pid=pid) 

604 if len(tapdiskList) > 1: #TODO: There might more than one minor for this blktap? 

605 raise xs_errors.XenError("TapdiskAlreadyRunning", "There is multiple minor for this tapdisk process") 

606 minor = tapdiskList[0]["minor"] 

607 return (pid, minor) 

608 raise xs_errors.XenError("TapdiskFailed", "No tapdisk process found for {}".format(path)) 

609 

610 @override 

611 def coalesceOnline(self, path: str) -> int: 

612 pid, minor = self._getTapdisk(path) 

613 logger = util.LoggerCounter(10) 

614 undefined_count = 0 

615 

616 try: 

617 TapCtl.commit(pid, minor, QCOW2_TYPE, path) 

618 # We need to wait for query to return concluded 

619 # We are technically ininterruptible since being interrupted will only stop checking if the job is done. 

620 # We need to call `tap-ctl cancel` if we are interrupted, it is done in cleanup.py code. 

621 except TapCtl.CommandFailure: 

622 util.SMlog(f"Commit command failed on tapdisk instance {pid} (m: {minor}). Raising...") 

623 raise 

624 

625 try: 

626 status, nb, total = TapCtl.query(pid, minor) 

627 while status != "concluded": 

628 if status == "undefined": 

629 util.SMlog("Tapdisk {} (m: {}) coalesce status undefined for {}".format(pid, minor, path)) 

630 undefined_count += 1 

631 if undefined_count >= MAX_UNDEFINED_QUERY_ATTEMPTS: 

632 raise xs_errors.XenError("TapdiskFailed", f"We had {undefined_count} undefined while waiting for online coalesce of {path} on tapdisk {pid} (m: {minor})") 

633 

634 time.sleep(1) 

635 status, nb, total = TapCtl.query(pid, minor, quiet=True) 

636 logger.log("Got status {} for tapdisk {} (m: {})".format(status, pid, minor)) 

637 

638 if nb < total: 

639 # The coalesce didn't finish because we might have been aborted. 

640 raise xs_errors.XenError("TapdiskFailed", 

641 f"Online coalesce incomplete on tapdisk {pid} (m: {minor}) for {path}: {nb}/{total} bytes") 

642 return nb 

643 except TapCtl.CommandFailure: 

644 util.SMlog(f"Query command failed on tapdisk instance {pid} (m: {minor}). Raising...") 

645 raise 

646 

647 @override 

648 def cancelCoalesceOnline(self, path: str) -> None: 

649 pid, minor = self._getTapdisk(path) 

650 

651 try: 

652 TapCtl.cancel_commit(pid, minor) 

653 except TapCtl.CommandFailure: 

654 util.SMlog("Cancel command failed on tapdisk instance {}. Raising...".format(pid)) 

655 raise 

656 

657 @override 

658 def coalesce(self, path: str) -> int: 

659 # -d on commit make it not empty the original image since we don't intend to keep it 

660 cmd = [QEMU_IMG, "commit", "-f", QCOW2_TYPE, path, "-d"] 

661 ret = cast(str, self._ioretry(cmd)) # Allows to parse for byte coalesced, our qemu-img is supposed to be patched to output it. 

662 lines = ret.splitlines() 

663 if re.match("Image committed.", lines[-1]): 

664 res_line = lines[-2] 

665 else: 

666 res_line = lines[-1] 

667 

668 results = re.match(r"\((\d+)/(\d+)\)", res_line) 

669 if results: 

670 committed_bytes = int(results.group(1)) 

671 return committed_bytes 

672 raise xs_errors.XenError("TapdiskFailed", "Couldn't get commited size from qemu-img commit call") # TODO: We might not want to raise in this case, it would break if the qemu-img called isn't modified to print the coalesce result even if it succeeded in coalesceing 

673 

674 @override 

675 def create(self, path: str, size: int, static: bool, msize: int = 0, block_size: Optional[int] = None) -> None: 

676 cmd = [QEMU_IMG, "create", "-q", "-f", QCOW2_TYPE, path, str(size)] 

677 if static: 

678 cmd.extend(["-o", "preallocation=full"]) 

679 if block_size: 

680 cmd.extend(["-o", f"cluster_size={str(block_size)}"]) 

681 self._ioretry(cmd) 

682 self.setHidden(path, False) #We add hidden header at creation 

683 

684 @override 

685 def snapshot( 

686 self, 

687 path: str, 

688 parent: str, 

689 parentRaw: bool, 

690 msize: int = 0, 

691 checkEmpty: bool = True, 

692 is_mirror_image: bool = False 

693 ) -> None: 

694 # TODO: msize, it's use to preallocate metadata, could we honor this too? 

695 # TODO: checkEmpty? If it is False, then the parent could be empty and should still be used for snapshot 

696 # But if True, if the parent is empty, we do what? vhd would just use the parent of parent as base, should we emulate this behavior? 

697 

698 cmd = [QEMU_IMG, "create", "-q"] 

699 

700 if parentRaw: 

701 parent_type = RAW_TYPE 

702 cluster_size = QCOW2_DEFAULT_CLUSTER_SIZE 

703 extended_l2 = False 

704 else: 

705 parent_type = QCOW2_TYPE 

706 cluster_size = self.getBlockSize(parent) 

707 extended_l2 = self._has_extended_l2(parent) 

708 args = ["-f", QCOW2_TYPE, "-F", parent_type, "-b", parent] 

709 

710 if is_mirror_image: 

711 # is_mirror_image override the cluster size to ensure that we have a write of 512b to avoid having to read the parent during a migration. 

712 # This is needed because the blkif blocksize is only 512b, as such it will try to only write blocks smaller than the cluster size. 

713 # To write a smaller block, we would need to read the parent image cluster then change the 512b block. 

714 # The parent being empty during the mirroring phase, reading from it would read zeros and corrupt the cluster. 

715 # It also enable extended_l2 for this purpose, this is only done in the snapshot used for the mirror, this configuration will be lost when coalesced in its parent. 

716 # Ensuring we go back to a better cluster_size for performance reasons. 

717 # This limit our images max size to 64TiB. 

718 cluster_size = 16 * 1024 # 16KiB 

719 extended_l2 = True 

720 

721 args.extend(["-o", f"cluster_size={cluster_size},extended_l2={self._convert_bool_to_qemu_option_value(extended_l2)}"]) 

722 cmd.extend(args) 

723 cmd.append(path) 

724 

725 self._ioretry(cmd) 

726 self.setHidden(path, False) #We add hidden header at creation 

727 

728 @override 

729 def canSnapshotRaw(self, size: int) -> bool: 

730 return size <= MAX_QCOW_SIZE 

731 

732 @override 

733 def check( 

734 self, 

735 path: str, 

736 ignoreMissingFooter: bool = False, 

737 fast: bool = False 

738 ) -> CowUtil.CheckResult: 

739 cmd = [QEMU_IMG, "check", "-q", path] 

740 try: 

741 self._ioretry(cmd) 

742 return CowUtil.CheckResult.Success 

743 except util.CommandException as e: 

744 if e.code in (errno.EROFS, errno.EMEDIUMTYPE): 

745 return CowUtil.CheckResult.Unavailable 

746 # 1/EPERM is error in internal during check 

747 # 2/ENOENT is QCOW corrupted 

748 # 3/ESRCH is QCow has leaked clusters 

749 # 63/ENOSR is check unavailable on this image type 

750 return CowUtil.CheckResult.Fail 

751 

752 @override 

753 def revert(self, path: str, jFile: str) -> None: 

754 pass #Used to get back from a failed operation using a journal, NOOP for qcow2 

755 

756 @override 

757 def repair(self, path: str) -> None: 

758 cmd = [QEMU_IMG, "check", "-q", "-f", QCOW2_TYPE, "-r", "all", path] 

759 self._ioretry(cmd) 

760 

761 @override 

762 def validateAndRoundImageSize(self, size: int) -> int: 

763 if size < 0 or size > MAX_QCOW_SIZE: 

764 raise xs_errors.XenError( 

765 "VDISize", 

766 opterr="VDI size must be between 1 MB and {} MB".format(MAX_QCOW_SIZE // (1024 * 1024)) 

767 ) 

768 

769 return util.roundup(QCOW2_DEFAULT_CLUSTER_SIZE, size) 

770 

771 @override 

772 def getKeyHash(self, path: str) -> Optional[str]: 

773 pass 

774 

775 @override 

776 def setKey(self, path: str, key_hash: str) -> None: 

777 pass 

778 

779 @override 

780 def isCoalesceableOnRemote(self) -> bool: 

781 return True