Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1#!/usr/bin/env python3 

2# 

3# Copyright (C) 2020 Vates SAS - ronan.abhamon@vates.fr 

4# 

5# This program is free software: you can redistribute it and/or modify 

6# it under the terms of the GNU General Public License as published by 

7# the Free Software Foundation, either version 3 of the License, or 

8# (at your option) any later version. 

9# This program is distributed in the hope that it will be useful, 

10# but WITHOUT ANY WARRANTY; without even the implied warranty of 

11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

12# GNU General Public License for more details. 

13# 

14# You should have received a copy of the GNU General Public License 

15# along with this program. If not, see <https://www.gnu.org/licenses/>. 

16# 

17 

18from sm_typing import ( 

19 Any, 

20 Dict, 

21 List, 

22 cast, 

23 override, 

24) 

25 

26import json 

27import linstor 

28import os.path 

29import re 

30import shutil 

31import socket 

32import stat 

33import time 

34import util 

35import uuid 

36from datetime import datetime 

37from pathlib import Path 

38import contextlib 

39import zipfile 

40 

41# Persistent prefix to add to RAW persistent volumes. 

42PERSISTENT_PREFIX = 'xcp-persistent-' 

43 

44# Contains the data of the "/var/lib/linstor" directory. 

45DATABASE_VOLUME_NAME = PERSISTENT_PREFIX + 'database' 

46DATABASE_SIZE = 1 << 30 # 1GB. 

47DATABASE_PATH = '/var/lib/linstor' 

48DATABASE_MKFS = 'mkfs.ext4' 

49DATABASE_BACKUP_DIR_MAIN = Path(DATABASE_PATH) 

50DATABASE_BACKUP_DIR_SPARE = Path("/var/lib/linstor.d/db-backups") 

51DATABASE_BACKUP_NAME_FORMAT = "linstor_database_backup-{}-{}" 

52DATABASE_BACKUP_RETENTION = 10 

53DATABASE_BACKUP_DATE_FORMAT = "%Y%m%d_%H%M%S" 

54LINSTOR_SATELLITE_PORT = 3366 

55 

56REG_DRBDADM_PRIMARY = re.compile("([^\\s]+)\\s+role:Primary") 

57REG_DRBDSETUP_IP = re.compile('[^\\s]+\\s+(.*):.*$') 

58 

59DRBD_BY_RES_PATH = '/dev/drbd/by-res/' 

60DRBD_OPENERS_PATH_FORMAT = '/sys/kernel/debug/drbd/resources/{}/volumes/{}/openers' 

61 

62PLUGIN = 'linstor-manager' 

63 

64LinstorLocalVolumeOpeners = Dict[str, Dict[str, Any]] 

65LinstorVolumeOpeners = Dict[str, LinstorLocalVolumeOpeners] 

66 

67# ============================================================================== 

68 

69def has_local_volume_openers(resource_name, volume) -> bool: 

70 if not resource_name or volume is None: 

71 raise Exception('Cannot check for DRBD openers without resource name and/or volume.') 

72 

73 path = DRBD_OPENERS_PATH_FORMAT.format(resource_name, volume) 

74 openers_char = '' 

75 

76 try: 

77 with open(path, 'r') as openers: 

78 # Read a single character to check if there is any opener listed. 

79 # If we get nothing back, then the file is empty meaning there are 

80 # no openers. 

81 openers_char = openers.read(1) 

82 except FileNotFoundError: 

83 return False 

84 

85 return openers_char != '' 

86 

87def get_local_volume_openers(resource_name, volume) -> LinstorLocalVolumeOpeners: 

88 if not resource_name or volume is None: 

89 raise Exception('Cannot get DRBD openers without resource name and/or volume.') 

90 

91 path = DRBD_OPENERS_PATH_FORMAT.format(resource_name, volume) 

92 

93 with open(path, 'r') as openers: 

94 # Not a big cost, so read all lines directly. 

95 lines = openers.readlines() 

96 

97 result = {} 

98 

99 opener_re = re.compile('(.*)\\s+([0-9]+)\\s+([0-9]+)') 

100 for line in lines: 

101 match = opener_re.match(line) 

102 assert match 

103 

104 groups = match.groups() 

105 process_name = groups[0] 

106 pid = groups[1] 

107 open_duration_ms = groups[2] 

108 

109 try: 

110 cmdline = util.get_process_cmdline(int(pid)) 

111 except Exception as e: 

112 util.SMlog(f"Failed to get command line of `{pid}`: {e}") 

113 cmdline = [] 

114 

115 result[pid] = { 

116 'process-name': process_name, 

117 'open-duration': open_duration_ms, 

118 'cmdline': cmdline 

119 } 

120 

121 return cast(LinstorLocalVolumeOpeners, json.dumps(result)) 

122 

123def get_all_volume_openers(resource_name, volume) -> LinstorVolumeOpeners: 

124 PLUGIN_CMD = 'getDrbdOpeners' 

125 

126 volume = str(volume) 

127 openers = {} 

128 

129 with util.ApiSession("SM-get-linstor-volume-openers") as session: 

130 hosts = session.xenapi.host.get_all_records() 

131 for host_ref, host_record in hosts.items(): 

132 node_name = host_record['hostname'] 

133 try: 

134 if not session.xenapi.host_metrics.get_record( 

135 host_record['metrics'] 

136 )['live']: 

137 # Ensure we call plugin on online hosts only. 

138 continue 

139 

140 openers[node_name] = json.loads( 

141 session.xenapi.host.call_plugin(host_ref, PLUGIN, PLUGIN_CMD, { 

142 'resourceName': resource_name, 

143 'volume': volume 

144 }) 

145 ) 

146 except Exception as e: 

147 util.SMlog('Failed to get openers of `{}` on `{}`: {}'.format( 

148 resource_name, node_name, e 

149 )) 

150 

151 return openers 

152 

153 

154# ============================================================================== 

155 

156def round_up(value, divisor): 

157 assert divisor 

158 divisor = int(divisor) 

159 return ((int(value) + divisor - 1) // divisor) * divisor 

160 

161 

162def round_down(value, divisor): 

163 assert divisor 

164 value = int(value) 

165 return value - (value % int(divisor)) 

166 

167 

168# ============================================================================== 

169 

170def _get_controller_addresses() -> List[str]: 

171 try: 

172 (ret, stdout, stderr) = util.doexec([ 

173 "/usr/sbin/ss", "-tnH", "state", "established", f"( sport = :{LINSTOR_SATELLITE_PORT} )" 

174 ]) 

175 if ret == 0: 

176 return [ 

177 line.split()[3].rsplit(":", 1)[0] 

178 for line in stdout.splitlines() 

179 ] 

180 util.SMlog(f"Unexpected code {ret}: {stderr}") 

181 except Exception as e: 

182 util.SMlog(f"Unable to get controller addresses: {e}") 

183 return [] 

184 

185def _get_controller_uri() -> str: 

186 # TODO: Check that an IP address from the current pool is returned. 

187 addresses = _get_controller_addresses() 

188 return "linstor://" + addresses[0] if addresses else "" 

189 

190def get_controller_uri(): 

191 retries = 0 

192 while True: 

193 uri = _get_controller_uri() 

194 if uri: 

195 return uri 

196 

197 retries += 1 

198 if retries >= 30: 

199 break 

200 time.sleep(1) 

201 

202 

203def get_controller_node_name(): 

204 PLUGIN_CMD = 'hasControllerRunning' 

205 

206 (ret, stdout, stderr) = util.doexec([ 

207 'drbdadm', 'status', DATABASE_VOLUME_NAME 

208 ]) 

209 

210 if ret == 0: 

211 if stdout.startswith('{} role:Primary'.format(DATABASE_VOLUME_NAME)): 

212 return 'localhost' 

213 

214 res = REG_DRBDADM_PRIMARY.search(stdout) 

215 if res: 

216 return res.groups()[0] 

217 

218 session = util.timeout(5, util.ApiSession, "SM-get-linstor-controller-node-name").session 

219 

220 for host_ref, host_record in session.xenapi.host.get_all_records().items(): 

221 node_name = host_record['hostname'] 

222 try: 

223 if not session.xenapi.host_metrics.get_record( 

224 host_record['metrics'] 

225 )['live']: 

226 continue 

227 

228 if util.strtobool(session.xenapi.host.call_plugin( 

229 host_ref, PLUGIN, PLUGIN_CMD, {} 

230 )): 

231 return node_name 

232 except Exception as e: 

233 util.SMlog('Failed to call plugin to get controller on `{}`: {}'.format( 

234 node_name, e 

235 )) 

236 

237 

238def demote_drbd_resource(node_name, resource_name): 

239 PLUGIN_CMD = 'demoteDrbdResource' 

240 

241 session = util.timeout(5, util.ApiSession, "SM-demote-drbd-resource").session 

242 

243 for host_ref, host_record in session.xenapi.host.get_all_records().items(): 

244 if host_record['hostname'] != node_name: 

245 continue 

246 

247 try: 

248 session.xenapi.host.call_plugin( 

249 host_ref, PLUGIN, PLUGIN_CMD, {'resource_name': resource_name} 

250 ) 

251 except Exception as e: 

252 util.SMlog('Failed to demote resource `{}` on `{}`: {}'.format( 

253 resource_name, node_name, e 

254 )) 

255 raise Exception( 

256 'Can\'t demote resource `{}`, unable to find node `{}`' 

257 .format(resource_name, node_name) 

258 ) 

259 

260# ============================================================================== 

261 

262class LinstorVolumeManagerError(Exception): 

263 ERR_GENERIC = 0, 

264 ERR_VOLUME_EXISTS = 1, 

265 ERR_VOLUME_NOT_EXISTS = 2, 

266 ERR_VOLUME_DESTROY = 3, 

267 ERR_GROUP_NOT_EXISTS = 4, 

268 ERR_VOLUME_IN_USE = 5, 

269 ERR_REPAIR_REQUIRED = 6, 

270 

271 def __init__(self, message, code=ERR_GENERIC): 

272 super(LinstorVolumeManagerError, self).__init__(message) 

273 self._code = code 

274 

275 @property 

276 def code(self): 

277 return self._code 

278 

279class LinstorDatabaseBackupError(Exception): 

280 pass 

281 

282# ============================================================================== 

283 

284# Note: 

285# If a storage pool is not accessible after a network change: 

286# linstor node interface modify <NODE> default --ip <IP> 

287 

288 

289class LinstorVolumeManager(object): 

290 """ 

291 API to manager LINSTOR volumes in XCP-ng. 

292 A volume in this context is a physical part of the storage layer. 

293 """ 

294 

295 __slots__ = ( 

296 '_linstor', '_uri', '_logger', '_redundancy', 

297 '_base_group_name', '_group_name', '_ha_group_name', 

298 '_volumes', '_storage_pools', '_storage_pools_time', 

299 '_kv_cache', '_resource_cache', '_volume_info_cache', 

300 '_kv_cache_dirty', '_resource_cache_dirty', '_volume_info_cache_dirty', 

301 '_resources_info_cache', 

302 ) 

303 

304 DEV_ROOT_PATH = DRBD_BY_RES_PATH 

305 

306 # Default sector size. 

307 BLOCK_SIZE = 512 

308 

309 # List of volume properties. 

310 PROP_METADATA = 'metadata' 

311 PROP_NOT_EXISTS = 'not-exists' 

312 PROP_VOLUME_NAME = 'volume-name' 

313 PROP_IS_READONLY_TIMESTAMP = 'readonly-timestamp' 

314 

315 # A volume can only be locked for a limited duration. 

316 # The goal is to give enough time to slaves to execute some actions on 

317 # a device before an UUID update or a coalesce for example. 

318 # Expiration is expressed in seconds. 

319 LOCKED_EXPIRATION_DELAY = 1 * 60 

320 

321 # Used when volume uuid is being updated. 

322 PROP_UPDATING_UUID_SRC = 'updating-uuid-src' 

323 

324 # States of property PROP_NOT_EXISTS. 

325 STATE_EXISTS = '0' 

326 STATE_NOT_EXISTS = '1' 

327 STATE_CREATING = '2' 

328 

329 # Property namespaces. 

330 NAMESPACE_SR = 'xcp/sr' 

331 NAMESPACE_VOLUME = 'xcp/volume' 

332 

333 # Regex to match properties. 

334 REG_PROP = '^([^/]+)/{}$' 

335 

336 REG_METADATA = re.compile(REG_PROP.format(PROP_METADATA)) 

337 REG_NOT_EXISTS = re.compile(REG_PROP.format(PROP_NOT_EXISTS)) 

338 REG_VOLUME_NAME = re.compile(REG_PROP.format(PROP_VOLUME_NAME)) 

339 REG_UPDATING_UUID_SRC = re.compile(REG_PROP.format(PROP_UPDATING_UUID_SRC)) 

340 

341 # Prefixes of SR/VOLUME in the LINSTOR DB. 

342 # A LINSTOR (resource, group, ...) name cannot start with a number. 

343 # So we add a prefix behind our SR/VOLUME uuids. 

344 PREFIX_SR = 'xcp-sr-' 

345 PREFIX_HA = 'xcp-ha-' 

346 PREFIX_VOLUME = 'xcp-volume-' 

347 

348 # Limit request number when storage pool info is asked, we fetch 

349 # the current pool status after N elapsed seconds. 

350 STORAGE_POOLS_FETCH_INTERVAL = 15 

351 

352 @staticmethod 

353 def default_logger(*args): 

354 print(args) 

355 

356 # -------------------------------------------------------------------------- 

357 # API. 

358 # -------------------------------------------------------------------------- 

359 

360 class VolumeInfo(object): 

361 __slots__ = ( 

362 'name', 

363 'allocated_size', # Allocated size, place count is not used. 

364 'virtual_size', # Total virtual available size of this volume 

365 # (i.e. the user size at creation). 

366 'diskful' # Array of nodes that have a diskful volume. 

367 ) 

368 

369 def __init__(self, name): 

370 self.name = name 

371 self.allocated_size = 0 

372 self.virtual_size = 0 

373 self.diskful = [] 

374 

375 @override 

376 def __repr__(self) -> str: 

377 return 'VolumeInfo("{}", {}, {}, {})'.format( 

378 self.name, self.allocated_size, self.virtual_size, 

379 self.diskful 

380 ) 

381 

382 # -------------------------------------------------------------------------- 

383 

384 def __init__( 

385 self, uri, group_name, repair=False, logger=default_logger.__func__, 

386 attempt_count=30 

387 ): 

388 """ 

389 Create a new LinstorVolumeManager object. 

390 :param str uri: URI to communicate with the LINSTOR controller. 

391 :param str group_name: The SR group name to use. 

392 :param bool repair: If true we try to remove bad volumes due to a crash 

393 or unexpected behavior. 

394 :param function logger: Function to log messages. 

395 :param int attempt_count: Number of attempts to join the controller. 

396 """ 

397 

398 self._uri = uri 

399 self._linstor = self._create_linstor_instance( 

400 uri, attempt_count=attempt_count 

401 ) 

402 

403 

404 mismatched_nodes = [ 

405 node for node in self._linstor.node_list().pop().nodes if node.connection_status == "VERSION_MISMATCH" 

406 ] 

407 

408 if mismatched_nodes: 

409 raise LinstorVolumeManagerError( 

410 "Some linstor nodes are not using the same version. " + 

411 f"Incriminated nodes are: {','.join([node.name for node in mismatched_nodes])}" 

412 ) 

413 

414 self._base_group_name = group_name 

415 

416 # Ensure group exists. 

417 group_name = self.build_group_name(group_name) 

418 groups = self._linstor.resource_group_list_raise([group_name]).resource_groups 

419 if not groups: 

420 raise LinstorVolumeManagerError( 

421 'Unable to find `{}` Linstor SR'.format(group_name) 

422 ) 

423 

424 # Ok. ;) 

425 self._logger = logger 

426 self._redundancy = groups[0].select_filter.place_count 

427 self._group_name = group_name 

428 self._ha_group_name = self._build_ha_group_name(self._base_group_name) 

429 self._volumes = set() 

430 self._storage_pools_time = 0 

431 

432 # To increase performance and limit request count to LINSTOR services, 

433 # we use caches. 

434 self._resource_cache = None 

435 self._resource_cache_dirty = True 

436 self._volume_info_cache = None 

437 self._volume_info_cache_dirty = True 

438 self._resources_info_cache = None 

439 

440 # While building the volume list, an error might be thrown if repairing 

441 # the SR is required (e.g. a volume is in in the middle of having its 

442 # UUID being updated). 

443 # This may only be a transient state, so we can retry multiple times to 

444 # see if we reach a healthy state. 

445 retry_count = 0 

446 while True: 

447 try: 

448 self._kv_cache = self._create_kv_cache() 

449 self._build_volumes(repair=repair) 

450 

451 break 

452 except LinstorVolumeManagerError as e: 

453 if e.code != LinstorVolumeManagerError.ERR_REPAIR_REQUIRED or retry_count >= 10: 

454 raise 

455 

456 retry_count += 1 

457 self._logger( 

458 f'Cannot build volume list ({e}). Retry number: {retry_count}' 

459 ) 

460 

461 time.sleep(0.1) 

462 

463 @property 

464 def uri(self) -> str: 

465 return self._uri 

466 

467 @property 

468 def native_client(self) -> "linstor.Linstor": 

469 return self._linstor 

470 

471 @property 

472 def group_name(self): 

473 """ 

474 Give the used group name. 

475 :return: The group name. 

476 :rtype: str 

477 """ 

478 return self._base_group_name 

479 

480 @property 

481 def redundancy(self): 

482 """ 

483 Give the used redundancy. 

484 :return: The redundancy. 

485 :rtype: int 

486 """ 

487 return self._redundancy 

488 

489 @property 

490 def volumes(self): 

491 """ 

492 Give the volumes uuid set. 

493 :return: The volumes uuid set. 

494 :rtype: set(str) 

495 """ 

496 return self._volumes 

497 

498 @property 

499 def max_volume_size_allowed(self): 

500 """ 

501 Give the max volume size currently available in B. 

502 :return: The current size. 

503 :rtype: int 

504 """ 

505 

506 candidates = self._find_best_size_candidates() 

507 if not candidates: 

508 raise LinstorVolumeManagerError( 

509 'Failed to get max volume size allowed' 

510 ) 

511 

512 size = candidates[0].max_volume_size 

513 if size < 0: 

514 raise LinstorVolumeManagerError( 

515 'Invalid max volume size allowed given: {}'.format(size) 

516 ) 

517 return self.round_down_volume_size(size * 1024) 

518 

519 @property 

520 def physical_size(self): 

521 """ 

522 Give the total physical size of the SR. 

523 :return: The physical size. 

524 :rtype: int 

525 """ 

526 return self._compute_size('total_capacity') 

527 

528 @property 

529 def physical_free_size(self): 

530 """ 

531 Give the total free physical size of the SR. 

532 :return: The physical free size. 

533 :rtype: int 

534 """ 

535 return self._compute_size('free_capacity') 

536 

537 @property 

538 def allocated_volume_size(self): 

539 """ 

540 Give the allocated size for all volumes. The place count is not 

541 used here. When thick lvm is used, the size for one volume should 

542 be equal to the virtual volume size. With thin lvm, the size is equal 

543 or lower to the volume size. 

544 :return: The allocated size of all volumes. 

545 :rtype: int 

546 """ 

547 

548 # Paths: /res_name/vol_number/size 

549 sizes = {} 

550 

551 for resource in self._get_resource_cache().resources: 

552 if resource.name not in sizes: 

553 current = sizes[resource.name] = {} 

554 else: 

555 current = sizes[resource.name] 

556 

557 for volume in resource.volumes: 

558 # We ignore diskless pools of the form "DfltDisklessStorPool". 

559 if volume.storage_pool_name != self._group_name: 

560 continue 

561 

562 allocated_size = max(volume.allocated_size, 0) 

563 current_allocated_size = current.get(volume.number) or -1 

564 if allocated_size > current_allocated_size: 

565 current[volume.number] = allocated_size 

566 

567 total_size = 0 

568 for volumes in sizes.values(): 

569 for size in volumes.values(): 

570 total_size += size 

571 

572 return total_size * 1024 

573 

574 def get_min_physical_size(self): 

575 """ 

576 Give the minimum physical size of the SR. 

577 I.e. the size of the smallest disk + the number of pools. 

578 :return: The physical min size. 

579 :rtype: tuple(int, int) 

580 """ 

581 size = None 

582 pool_count = 0 

583 for pool in self._get_storage_pools(force=True): 

584 space = pool.free_space 

585 if space: 

586 pool_count += 1 

587 current_size = space.total_capacity 

588 if current_size < 0: 

589 raise LinstorVolumeManagerError( 

590 'Failed to get pool total_capacity attr of `{}`' 

591 .format(pool.node_name) 

592 ) 

593 if size is None or current_size < size: 

594 size = current_size 

595 return (pool_count, (size or 0) * 1024) 

596 

597 @property 

598 def metadata(self): 

599 """ 

600 Get the metadata of the SR. 

601 :return: Dictionary that contains metadata. 

602 :rtype: dict(str, dict) 

603 """ 

604 

605 sr_properties = self._get_sr_properties() 

606 metadata = sr_properties.get(self.PROP_METADATA) 

607 if metadata is not None: 

608 metadata = json.loads(metadata) 

609 if isinstance(metadata, dict): 

610 return metadata 

611 raise LinstorVolumeManagerError( 

612 'Expected dictionary in SR metadata: {}'.format( 

613 self._group_name 

614 ) 

615 ) 

616 

617 return {} 

618 

619 @metadata.setter 

620 def metadata(self, metadata): 

621 """ 

622 Set the metadata of the SR. 

623 :param dict metadata: Dictionary that contains metadata. 

624 """ 

625 

626 assert isinstance(metadata, dict) 

627 sr_properties = self._get_sr_properties() 

628 sr_properties[self.PROP_METADATA] = json.dumps(metadata) 

629 

630 @property 

631 def disconnected_hosts(self): 

632 """ 

633 Get the list of disconnected hosts. 

634 :return: Set that contains disconnected hosts. 

635 :rtype: set(str) 

636 """ 

637 

638 disconnected_hosts = set() 

639 for pool in self._get_storage_pools(): 

640 for report in pool.reports: 

641 if report.ret_code & linstor.consts.WARN_NOT_CONNECTED == \ 

642 linstor.consts.WARN_NOT_CONNECTED: 

643 disconnected_hosts.add(pool.node_name) 

644 break 

645 return disconnected_hosts 

646 

647 def check_volume_exists(self, volume_uuid): 

648 """ 

649 Check if a volume exists in the SR. 

650 :return: True if volume exists. 

651 :rtype: bool 

652 """ 

653 return volume_uuid in self._volumes 

654 

655 def create_volume( 

656 self, 

657 volume_uuid, 

658 size, 

659 persistent=True, 

660 volume_name=None, 

661 high_availability=False 

662 ): 

663 """ 

664 Create a new volume on the SR. 

665 :param str volume_uuid: The volume uuid to use. 

666 :param int size: volume size in B. 

667 :param bool persistent: If false the volume will be unavailable 

668 on the next constructor call LinstorSR(...). 

669 :param str volume_name: If set, this name is used in the LINSTOR 

670 database instead of a generated name. 

671 :param bool high_availability: If set, the volume is created in 

672 the HA group. 

673 :return: The current device path of the volume. 

674 :rtype: str 

675 """ 

676 

677 self._logger('Creating LINSTOR volume {}...'.format(volume_uuid)) 

678 if not volume_name: 

679 volume_name = self.build_volume_name(util.gen_uuid()) 

680 volume_properties = self._create_volume_with_properties( 

681 volume_uuid, 

682 volume_name, 

683 size, 

684 True, # place_resources 

685 high_availability 

686 ) 

687 

688 # Volume created! Now try to find the device path. 

689 try: 

690 self._logger( 

691 'Find device path of LINSTOR volume {}...'.format(volume_uuid) 

692 ) 

693 device_path = self._find_device_path(volume_uuid, volume_name) 

694 if persistent: 

695 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_EXISTS 

696 self._volumes.add(volume_uuid) 

697 self._logger( 

698 'LINSTOR volume {} created!'.format(volume_uuid) 

699 ) 

700 return device_path 

701 except Exception: 

702 # There is an issue to find the path. 

703 # At this point the volume has just been created, so force flag can be used. 

704 self._destroy_volume(volume_uuid, force=True) 

705 raise 

706 

707 def mark_volume_as_persistent(self, volume_uuid): 

708 """ 

709 Mark volume as persistent if created with persistent=False. 

710 :param str volume_uuid: The volume uuid to mark. 

711 """ 

712 

713 self._ensure_volume_exists(volume_uuid) 

714 

715 # Mark volume as persistent. 

716 volume_properties = self._get_volume_properties(volume_uuid) 

717 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_EXISTS 

718 

719 def destroy_volume(self, volume_uuid): 

720 """ 

721 Destroy a volume. 

722 :param str volume_uuid: The volume uuid to destroy. 

723 """ 

724 

725 self._ensure_volume_exists(volume_uuid) 

726 self.ensure_volume_is_not_locked(volume_uuid) 

727 

728 is_volume_in_use = any(node["in-use"] for node in self.get_resource_info(volume_uuid)["nodes"].values()) 

729 if is_volume_in_use: 

730 raise LinstorVolumeManagerError( 

731 f"Could not destroy volume `{volume_uuid}` as it is currently in use", 

732 LinstorVolumeManagerError.ERR_VOLUME_IN_USE 

733 ) 

734 

735 # Mark volume as destroyed. 

736 volume_properties = self._get_volume_properties(volume_uuid) 

737 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_NOT_EXISTS 

738 

739 try: 

740 self._volumes.remove(volume_uuid) 

741 self._destroy_volume(volume_uuid) 

742 except Exception as e: 

743 raise LinstorVolumeManagerError( 

744 str(e), 

745 LinstorVolumeManagerError.ERR_VOLUME_DESTROY 

746 ) 

747 

748 def lock_volume(self, volume_uuid, locked=True): 

749 """ 

750 Prevent modifications of the volume properties during 

751 "self.LOCKED_EXPIRATION_DELAY" seconds. The SR must be locked 

752 when used. This method is useful to attach/detach correctly a volume on 

753 a slave. Without it the GC can rename a volume, in this case the old 

754 volume path can be used by a slave... 

755 :param str volume_uuid: The volume uuid to protect/unprotect. 

756 :param bool locked: Lock/unlock the volume. 

757 """ 

758 

759 self._ensure_volume_exists(volume_uuid) 

760 

761 self._logger( 

762 '{} volume {} as locked'.format( 

763 'Mark' if locked else 'Unmark', 

764 volume_uuid 

765 ) 

766 ) 

767 

768 volume_properties = self._get_volume_properties(volume_uuid) 

769 if locked: 

770 volume_properties[ 

771 self.PROP_IS_READONLY_TIMESTAMP 

772 ] = str(time.time()) 

773 elif self.PROP_IS_READONLY_TIMESTAMP in volume_properties: 

774 volume_properties.pop(self.PROP_IS_READONLY_TIMESTAMP) 

775 

776 def ensure_volume_is_not_locked(self, volume_uuid, timeout=None): 

777 """ 

778 Ensure a volume is not locked. Wait if necessary. 

779 :param str volume_uuid: The volume uuid to check. 

780 :param int timeout: If the volume is always locked after the expiration 

781 of the timeout, an exception is thrown. 

782 """ 

783 return self.ensure_volume_list_is_not_locked([volume_uuid], timeout) 

784 

785 def ensure_volume_list_is_not_locked(self, volume_uuids, timeout=None): 

786 checked = set() 

787 for volume_uuid in volume_uuids: 

788 if volume_uuid in self._volumes: 

789 checked.add(volume_uuid) 

790 

791 if not checked: 

792 return 

793 

794 waiting = False 

795 

796 volume_properties = self._get_kv_cache() 

797 

798 start = time.time() 

799 while True: 

800 # Can't delete in for loop, use a copy of the list. 

801 remaining = checked.copy() 

802 for volume_uuid in checked: 

803 volume_properties.namespace = \ 

804 self._build_volume_namespace(volume_uuid) 

805 timestamp = volume_properties.get( 

806 self.PROP_IS_READONLY_TIMESTAMP 

807 ) 

808 if timestamp is None: 

809 remaining.remove(volume_uuid) 

810 continue 

811 

812 now = time.time() 

813 if now - float(timestamp) > self.LOCKED_EXPIRATION_DELAY: 

814 self._logger( 

815 'Remove readonly timestamp on {}'.format(volume_uuid) 

816 ) 

817 volume_properties.pop(self.PROP_IS_READONLY_TIMESTAMP) 

818 remaining.remove(volume_uuid) 

819 continue 

820 

821 if not waiting: 

822 self._logger( 

823 'Volume {} is locked, waiting...'.format(volume_uuid) 

824 ) 

825 waiting = True 

826 break 

827 

828 if not remaining: 

829 break 

830 checked = remaining 

831 

832 if timeout is not None and now - start > timeout: 

833 raise LinstorVolumeManagerError( 

834 'volume `{}` is locked and timeout has been reached' 

835 .format(volume_uuid), 

836 LinstorVolumeManagerError.ERR_VOLUME_NOT_EXISTS 

837 ) 

838 

839 # We must wait to use the volume. After that we can modify it 

840 # ONLY if the SR is locked to avoid bad reads on the slaves. 

841 time.sleep(1) 

842 volume_properties = self._create_kv_cache() 

843 

844 if waiting: 

845 self._logger('No volume locked now!') 

846 

847 def remove_volume_if_diskless(self, volume_uuid): 

848 """ 

849 Remove disless path from local node. 

850 :param str volume_uuid: The volume uuid to remove. 

851 """ 

852 

853 self._ensure_volume_exists(volume_uuid) 

854 

855 volume_properties = self._get_volume_properties(volume_uuid) 

856 volume_name = volume_properties.get(self.PROP_VOLUME_NAME) 

857 

858 node_name = socket.gethostname() 

859 

860 for resource in self._get_resource_cache().resources: 

861 if resource.name == volume_name and resource.node_name == node_name: 

862 if linstor.consts.FLAG_TIE_BREAKER in resource.flags: 

863 return 

864 break 

865 

866 if has_local_volume_openers(volume_name, 0): 

867 return 

868 

869 result = self._linstor.resource_delete_if_diskless( 

870 node_name=node_name, rsc_name=volume_name 

871 ) 

872 if not linstor.Linstor.all_api_responses_no_error(result): 

873 raise LinstorVolumeManagerError( 

874 'Unable to delete diskless path of `{}` on node `{}`: {}' 

875 .format(volume_name, node_name, ', '.join( 

876 [str(x) for x in result])) 

877 ) 

878 

879 def introduce_volume(self, volume_uuid): 

880 pass # TODO: Implement me. 

881 

882 def resize_volume(self, volume_uuid, new_size): 

883 """ 

884 Resize a volume. 

885 :param str volume_uuid: The volume uuid to resize. 

886 :param int new_size: New size in B. 

887 """ 

888 

889 volume_name = self.get_volume_name(volume_uuid) 

890 self.ensure_volume_is_not_locked(volume_uuid) 

891 new_size = self.round_up_volume_size(new_size) // 1024 

892 

893 # We can't resize anything until DRBD is up to date. 

894 # We wait here for 5min max and raise an easy to understand error for the user. 

895 # 5min is an arbitrary time, it's impossible to get a fit all situation value 

896 # and it's currently impossible to know how much time we have to wait 

897 # This is mostly an issue for thick provisioning, thin isn't affected. 

898 start_time = time.monotonic() 

899 try: 

900 self._linstor.resource_dfn_wait_synced(volume_name, wait_interval=1.0, timeout=60*5) 

901 except linstor.LinstorTimeoutError: 

902 raise LinstorVolumeManagerError( 

903 f"Volume resizing of `{volume_uuid}` from SR `{self._group_name}` is incomplete: timeout reached but it continues in background." 

904 ) 

905 util.SMlog(f"DRBD is up to date, syncing took {time.monotonic() - start_time}s") 

906 

907 result = self._linstor.volume_dfn_modify( 

908 rsc_name=volume_name, 

909 volume_nr=0, 

910 size=new_size 

911 ) 

912 

913 self._mark_resource_cache_as_dirty() 

914 

915 error_str = self._get_error_str(result) 

916 if error_str: 

917 raise LinstorVolumeManagerError( 

918 f"Could not resize volume `{volume_uuid}` from SR `{self._group_name}`: {error_str}" 

919 ) 

920 

921 def get_volume_name(self, volume_uuid): 

922 """ 

923 Get the name of a particular volume. 

924 :param str volume_uuid: The volume uuid of the name to get. 

925 :return: The volume name. 

926 :rtype: str 

927 """ 

928 

929 self._ensure_volume_exists(volume_uuid) 

930 volume_properties = self._get_volume_properties(volume_uuid) 

931 volume_name = volume_properties.get(self.PROP_VOLUME_NAME) 

932 if volume_name: 

933 return volume_name 

934 raise LinstorVolumeManagerError( 

935 'Failed to get volume name of {}'.format(volume_uuid) 

936 ) 

937 

938 def get_volume_size(self, volume_uuid): 

939 """ 

940 Get the size of a particular volume. 

941 :param str volume_uuid: The volume uuid of the size to get. 

942 :return: The volume size. 

943 :rtype: int 

944 """ 

945 

946 volume_name = self.get_volume_name(volume_uuid) 

947 dfns = self._linstor.resource_dfn_list_raise( 

948 query_volume_definitions=True, 

949 filter_by_resource_definitions=[volume_name] 

950 ).resource_definitions 

951 

952 size = dfns[0].volume_definitions[0].size 

953 if size < 0: 

954 raise LinstorVolumeManagerError( 

955 'Failed to get volume size of: {}'.format(volume_uuid) 

956 ) 

957 return size * 1024 

958 

959 def set_auto_promote_timeout(self, volume_uuid, timeout): 

960 """ 

961 Define the blocking time of open calls when a DRBD 

962 is already open on another host. 

963 :param str volume_uuid: The volume uuid to modify. 

964 """ 

965 

966 volume_name = self.get_volume_name(volume_uuid) 

967 result = self._linstor.resource_dfn_modify(volume_name, { 

968 'DrbdOptions/Resource/auto-promote-timeout': timeout 

969 }) 

970 error_str = self._get_error_str(result) 

971 if error_str: 

972 raise LinstorVolumeManagerError( 

973 'Could not change the auto promote timeout of `{}`: {}' 

974 .format(volume_uuid, error_str) 

975 ) 

976 

977 def set_drbd_ha_properties(self, volume_name, enabled=True): 

978 """ 

979 Set or not HA DRBD properties required by drbd-reactor and 

980 by specific volumes. 

981 :param str volume_name: The volume to modify. 

982 :param bool enabled: Enable or disable HA properties. 

983 """ 

984 

985 properties = { 

986 'DrbdOptions/auto-quorum': 'disabled', 

987 'DrbdOptions/Resource/auto-promote': 'no', 

988 'DrbdOptions/Resource/on-no-data-accessible': 'io-error', 

989 'DrbdOptions/Resource/on-no-quorum': 'io-error', 

990 'DrbdOptions/Resource/on-suspended-primary-outdated': 'force-secondary', 

991 'DrbdOptions/Resource/quorum': 'majority' 

992 } 

993 if enabled: 

994 result = self._linstor.resource_dfn_modify(volume_name, properties) 

995 else: 

996 result = self._linstor.resource_dfn_modify(volume_name, {}, delete_props=list(properties.keys())) 

997 

998 error_str = self._get_error_str(result) 

999 if error_str: 

1000 raise LinstorVolumeManagerError( 

1001 'Could not modify HA DRBD properties on volume `{}`: {}' 

1002 .format(volume_name, error_str) 

1003 ) 

1004 

1005 def get_volume_info(self, volume_uuid): 

1006 """ 

1007 Get the volume info of a particular volume. 

1008 :param str volume_uuid: The volume uuid of the volume info to get. 

1009 :return: The volume info. 

1010 :rtype: VolumeInfo 

1011 """ 

1012 

1013 volume_name = self.get_volume_name(volume_uuid) 

1014 try: 

1015 return self._get_volumes_info()[volume_name] 

1016 except KeyError: 

1017 raise LinstorVolumeManagerError( 

1018 f"Could not find info about volume `{volume_uuid}`, the resource definition has likely been deleted", 

1019 LinstorVolumeManagerError.ERR_VOLUME_NOT_EXISTS 

1020 ) 

1021 

1022 def get_device_path(self, volume_uuid): 

1023 """ 

1024 Get the dev path of a volume, create a diskless if necessary. 

1025 :param str volume_uuid: The volume uuid to get the dev path. 

1026 :return: The current device path of the volume. 

1027 :rtype: str 

1028 """ 

1029 

1030 volume_name = self.get_volume_name(volume_uuid) 

1031 return self._find_device_path(volume_uuid, volume_name) 

1032 

1033 def get_volume_uuid_from_device_path(self, device_path): 

1034 """ 

1035 Get the volume uuid of a device_path. 

1036 :param str device_path: The dev path to find the volume uuid. 

1037 :return: The volume uuid of the local device path. 

1038 :rtype: str 

1039 """ 

1040 

1041 expected_volume_name = \ 

1042 self.get_volume_name_from_device_path(device_path) 

1043 

1044 volume_names = self.get_volumes_with_name() 

1045 for volume_uuid, volume_name in volume_names.items(): 

1046 if volume_name == expected_volume_name: 

1047 return volume_uuid 

1048 

1049 raise LinstorVolumeManagerError( 

1050 'Unable to find volume uuid from dev path `{}`'.format(device_path) 

1051 ) 

1052 

1053 def get_volume_name_from_device_path(self, device_path): 

1054 """ 

1055 Get the volume name of a device_path. 

1056 :param str device_path: The dev path to find the volume name. 

1057 :return: The volume name of the device path. 

1058 :rtype: str 

1059 """ 

1060 

1061 # Assume that we have a path like this: 

1062 # - "/dev/drbd/by-res/xcp-volume-<UUID>/0" 

1063 # - "../xcp-volume-<UUID>/0" 

1064 if device_path.startswith(DRBD_BY_RES_PATH): 

1065 prefix_len = len(DRBD_BY_RES_PATH) 

1066 elif device_path.startswith('../'): 

1067 prefix_len = 3 

1068 else: 

1069 raise LinstorVolumeManagerError('Unexpected device path: `{}`'.format(device_path)) 

1070 

1071 res_name_end = device_path.find('/', prefix_len) 

1072 assert res_name_end != -1 

1073 return device_path[prefix_len:res_name_end] 

1074 

1075 def update_volume_uuid(self, volume_uuid, new_volume_uuid, force=False): 

1076 """ 

1077 Change the uuid of a volume. 

1078 :param str volume_uuid: The volume to modify. 

1079 :param str new_volume_uuid: The new volume uuid to use. 

1080 :param bool force: If true we doesn't check if volume_uuid is in the 

1081 volume list. I.e. the volume can be marked as deleted but the volume 

1082 can still be in the LINSTOR KV store if the deletion has failed. 

1083 In specific cases like "undo" after a failed clone we must rename a bad 

1084 deleted VDI. 

1085 """ 

1086 

1087 self._logger( 

1088 'Trying to update volume UUID {} to {}...' 

1089 .format(volume_uuid, new_volume_uuid) 

1090 ) 

1091 assert volume_uuid != new_volume_uuid, 'can\'t update volume UUID, same value' 

1092 

1093 if not force: 

1094 self._ensure_volume_exists(volume_uuid) 

1095 self.ensure_volume_is_not_locked(volume_uuid) 

1096 

1097 if new_volume_uuid in self._volumes: 

1098 raise LinstorVolumeManagerError( 

1099 'Volume `{}` already exists'.format(new_volume_uuid), 

1100 LinstorVolumeManagerError.ERR_VOLUME_EXISTS 

1101 ) 

1102 

1103 volume_properties = self._get_volume_properties(volume_uuid) 

1104 if volume_properties.get(self.PROP_UPDATING_UUID_SRC): 

1105 raise LinstorVolumeManagerError( 

1106 'Cannot update volume uuid {}: invalid state' 

1107 .format(volume_uuid) 

1108 ) 

1109 

1110 # 1. Copy in temp variables metadata and volume_name. 

1111 metadata = volume_properties.get(self.PROP_METADATA) 

1112 volume_name = volume_properties.get(self.PROP_VOLUME_NAME) 

1113 

1114 # 2. Switch to new volume namespace. 

1115 volume_properties.namespace = self._build_volume_namespace( 

1116 new_volume_uuid 

1117 ) 

1118 

1119 if list(volume_properties.items()): 

1120 raise LinstorVolumeManagerError( 

1121 'Cannot update volume uuid {} to {}: ' 

1122 .format(volume_uuid, new_volume_uuid) + 

1123 'this last one is not empty' 

1124 ) 

1125 

1126 try: 

1127 # 3. Mark new volume properties with PROP_UPDATING_UUID_SRC. 

1128 # If we crash after that, the new properties can be removed 

1129 # properly. 

1130 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_NOT_EXISTS 

1131 volume_properties[self.PROP_UPDATING_UUID_SRC] = volume_uuid 

1132 

1133 # 4. Copy the properties. 

1134 # Note: On new volumes, during clone for example, the metadata 

1135 # may be missing. So we must test it to avoid this error: 

1136 # "None has to be a str/unicode, but is <type 'NoneType'>" 

1137 if metadata: 

1138 volume_properties[self.PROP_METADATA] = metadata 

1139 volume_properties[self.PROP_VOLUME_NAME] = volume_name 

1140 

1141 # 5. Ok! 

1142 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_EXISTS 

1143 except Exception as err: 

1144 try: 

1145 # Clear the new volume properties in case of failure. 

1146 assert volume_properties.namespace == \ 

1147 self._build_volume_namespace(new_volume_uuid) 

1148 volume_properties.clear() 

1149 except Exception as e: 

1150 self._logger( 

1151 'Failed to clear new volume properties: {} (ignoring...)' 

1152 .format(e) 

1153 ) 

1154 raise LinstorVolumeManagerError( 

1155 'Failed to copy volume properties: {}'.format(err) 

1156 ) 

1157 

1158 try: 

1159 # 6. After this point, it's ok we can remove the 

1160 # PROP_UPDATING_UUID_SRC property and clear the src properties 

1161 # without problems. 

1162 

1163 # 7. Switch to old volume namespace. 

1164 volume_properties.namespace = self._build_volume_namespace( 

1165 volume_uuid 

1166 ) 

1167 volume_properties.clear() 

1168 

1169 # 8. Switch a last time to new volume namespace. 

1170 volume_properties.namespace = self._build_volume_namespace( 

1171 new_volume_uuid 

1172 ) 

1173 volume_properties.pop(self.PROP_UPDATING_UUID_SRC) 

1174 except Exception as e: 

1175 raise LinstorVolumeManagerError( 

1176 'Failed to clear volume properties ' 

1177 'after volume uuid update: {}'.format(e) 

1178 ) 

1179 

1180 try: 

1181 self._volumes.remove(volume_uuid) 

1182 except KeyError: 

1183 # Can be missing if we are building the volume set attr AND 

1184 # we are processing a deleted resource. 

1185 assert force 

1186 

1187 self._volumes.add(new_volume_uuid) 

1188 

1189 self._logger( 

1190 'UUID update succeeded of {} to {}! (properties={})' 

1191 .format( 

1192 volume_uuid, new_volume_uuid, 

1193 self._get_filtered_properties(volume_properties) 

1194 ) 

1195 ) 

1196 

1197 def update_volume_name(self, volume_uuid, volume_name): 

1198 """ 

1199 Change the volume name of a volume. 

1200 :param str volume_uuid: The volume to modify. 

1201 :param str volume_name: The volume_name to use. 

1202 """ 

1203 

1204 self._ensure_volume_exists(volume_uuid) 

1205 self.ensure_volume_is_not_locked(volume_uuid) 

1206 if not volume_name.startswith(self.PREFIX_VOLUME): 

1207 raise LinstorVolumeManagerError( 

1208 'Volume name `{}` must be start with `{}`' 

1209 .format(volume_name, self.PREFIX_VOLUME) 

1210 ) 

1211 

1212 if volume_name not in self._fetch_resource_names(): 

1213 raise LinstorVolumeManagerError( 

1214 'Volume `{}` doesn\'t exist'.format(volume_name) 

1215 ) 

1216 

1217 volume_properties = self._get_volume_properties(volume_uuid) 

1218 volume_properties[self.PROP_VOLUME_NAME] = volume_name 

1219 

1220 def get_usage_states(self, volume_uuid): 

1221 """ 

1222 Check if a volume is currently used. 

1223 :param str volume_uuid: The volume uuid to check. 

1224 :return: A dictionary that contains states. 

1225 :rtype: dict(str, bool or None) 

1226 """ 

1227 

1228 states = {} 

1229 

1230 volume_name = self.get_volume_name(volume_uuid) 

1231 for resource_state in self._linstor.resource_list_raise( 

1232 filter_by_resources=[volume_name] 

1233 ).resource_states: 

1234 states[resource_state.node_name] = resource_state.in_use 

1235 

1236 return states 

1237 

1238 def get_volume_openers(self, volume_uuid) -> LinstorVolumeOpeners: 

1239 """ 

1240 Get openers of a volume. 

1241 :param str volume_uuid: The volume uuid to monitor. 

1242 :return: A dictionary that contains openers. 

1243 :rtype: dict(str, obj) 

1244 """ 

1245 return get_all_volume_openers(self.get_volume_name(volume_uuid), '0') 

1246 

1247 def get_volumes_with_name(self): 

1248 """ 

1249 Give a volume dictionary that contains names actually owned. 

1250 :return: A volume/name dict. 

1251 :rtype: dict(str, str) 

1252 """ 

1253 return self._get_volumes_by_property(self.REG_VOLUME_NAME) 

1254 

1255 def get_volumes_with_info(self): 

1256 """ 

1257 Give a volume dictionary that contains VolumeInfos. 

1258 :return: A volume/VolumeInfo dict. 

1259 :rtype: dict(str, VolumeInfo) 

1260 """ 

1261 

1262 volumes = {} 

1263 

1264 volume_names = self.get_volumes_with_name() 

1265 all_volume_info = self._get_volumes_info(volume_names) 

1266 for volume_uuid, volume_name in volume_names.items(): 

1267 if volume_name: 

1268 volume_info = all_volume_info.get(volume_name) 

1269 if volume_info: 

1270 volumes[volume_uuid] = volume_info 

1271 continue 

1272 

1273 # Well I suppose if this volume is not available, 

1274 # LINSTOR has been used directly without using this API. 

1275 volumes[volume_uuid] = self.VolumeInfo('') 

1276 

1277 return volumes 

1278 

1279 def get_volumes_with_metadata(self): 

1280 """ 

1281 Give a volume dictionary that contains metadata. 

1282 :return: A volume/metadata dict. 

1283 :rtype: dict(str, dict) 

1284 """ 

1285 

1286 volumes = {} 

1287 

1288 metadata = self._get_volumes_by_property(self.REG_METADATA) 

1289 for volume_uuid, volume_metadata in metadata.items(): 

1290 if volume_metadata: 

1291 volume_metadata = json.loads(volume_metadata) 

1292 if isinstance(volume_metadata, dict): 

1293 volumes[volume_uuid] = volume_metadata 

1294 continue 

1295 raise LinstorVolumeManagerError( 

1296 'Expected dictionary in volume metadata: {}' 

1297 .format(volume_uuid) 

1298 ) 

1299 

1300 volumes[volume_uuid] = {} 

1301 

1302 return volumes 

1303 

1304 def get_volume_metadata(self, volume_uuid): 

1305 """ 

1306 Get the metadata of a volume. 

1307 :return: Dictionary that contains metadata. 

1308 :rtype: dict 

1309 """ 

1310 

1311 self._ensure_volume_exists(volume_uuid) 

1312 volume_properties = self._get_volume_properties(volume_uuid) 

1313 metadata = volume_properties.get(self.PROP_METADATA) 

1314 if metadata: 

1315 metadata = json.loads(metadata) 

1316 if isinstance(metadata, dict): 

1317 return metadata 

1318 raise LinstorVolumeManagerError( 

1319 'Expected dictionary in volume metadata: {}' 

1320 .format(volume_uuid) 

1321 ) 

1322 return {} 

1323 

1324 def set_volume_metadata(self, volume_uuid, metadata): 

1325 """ 

1326 Set the metadata of a volume. 

1327 :param dict metadata: Dictionary that contains metadata. 

1328 """ 

1329 

1330 self._ensure_volume_exists(volume_uuid) 

1331 self.ensure_volume_is_not_locked(volume_uuid) 

1332 

1333 assert isinstance(metadata, dict) 

1334 volume_properties = self._get_volume_properties(volume_uuid) 

1335 volume_properties[self.PROP_METADATA] = json.dumps(metadata) 

1336 

1337 def update_volume_metadata(self, volume_uuid, metadata): 

1338 """ 

1339 Update the metadata of a volume. It modify only the given keys. 

1340 It doesn't remove unreferenced key instead of set_volume_metadata. 

1341 :param dict metadata: Dictionary that contains metadata. 

1342 """ 

1343 

1344 self._ensure_volume_exists(volume_uuid) 

1345 self.ensure_volume_is_not_locked(volume_uuid) 

1346 

1347 assert isinstance(metadata, dict) 

1348 volume_properties = self._get_volume_properties(volume_uuid) 

1349 

1350 current_metadata = json.loads( 

1351 volume_properties.get(self.PROP_METADATA, '{}') 

1352 ) 

1353 if not isinstance(metadata, dict): 

1354 raise LinstorVolumeManagerError( 

1355 'Expected dictionary in volume metadata: {}' 

1356 .format(volume_uuid) 

1357 ) 

1358 

1359 for key, value in metadata.items(): 

1360 current_metadata[key] = value 

1361 volume_properties[self.PROP_METADATA] = json.dumps(current_metadata) 

1362 

1363 def shallow_clone_volume(self, volume_uuid, clone_uuid, persistent=True): 

1364 """ 

1365 Clone a volume. Do not copy the data, this method creates a new volume 

1366 with the same size. 

1367 :param str volume_uuid: The volume to clone. 

1368 :param str clone_uuid: The cloned volume. 

1369 :param bool persistent: If false the volume will be unavailable 

1370 on the next constructor call LinstorSR(...). 

1371 :return: The current device path of the cloned volume. 

1372 :rtype: str 

1373 """ 

1374 

1375 volume_name = self.get_volume_name(volume_uuid) 

1376 self.ensure_volume_is_not_locked(volume_uuid) 

1377 

1378 # 1. Find ideal nodes + size to use. 

1379 ideal_node_names, size = self._get_volume_node_names_and_size( 

1380 volume_name 

1381 ) 

1382 if size <= 0: 

1383 raise LinstorVolumeManagerError( 

1384 'Invalid size of {} for volume `{}`'.format(size, volume_name) 

1385 ) 

1386 

1387 # 2. Create clone! 

1388 return self.create_volume(clone_uuid, size, persistent) 

1389 

1390 def remove_resourceless_volumes(self): 

1391 """ 

1392 Remove all volumes without valid or non-empty name 

1393 (i.e. without LINSTOR resource). It's different than 

1394 LinstorVolumeManager constructor that takes a `repair` param that 

1395 removes volumes with `PROP_NOT_EXISTS` to 1. 

1396 """ 

1397 

1398 resource_names = self._fetch_resource_names() 

1399 for volume_uuid, volume_name in self.get_volumes_with_name().items(): 

1400 if not volume_name or volume_name not in resource_names: 

1401 # Don't force, we can be sure of what's happening. 

1402 self.destroy_volume(volume_uuid) 

1403 

1404 def destroy(self): 

1405 """ 

1406 Destroy this SR. Object should not be used after that. 

1407 :param bool force: Try to destroy volumes before if true. 

1408 """ 

1409 

1410 # 1. Ensure volume list is empty. No cost. 

1411 if self._volumes: 

1412 raise LinstorVolumeManagerError( 

1413 'Cannot destroy LINSTOR volume manager: ' 

1414 'It exists remaining volumes' 

1415 ) 

1416 

1417 # 2. Fetch ALL resource names. 

1418 # This list may therefore contain volumes created outside 

1419 # the scope of the driver. 

1420 resource_names = self._fetch_resource_names(ignore_deleted=False) 

1421 try: 

1422 resource_names.remove(DATABASE_VOLUME_NAME) 

1423 except KeyError: 

1424 # Really strange to reach that point. 

1425 # Normally we always have the database volume in the list. 

1426 pass 

1427 

1428 # 3. Ensure the resource name list is entirely empty... 

1429 if resource_names: 

1430 raise LinstorVolumeManagerError( 

1431 'Cannot destroy LINSTOR volume manager: ' 

1432 'It exists remaining volumes (created externally or being deleted)' 

1433 ) 

1434 

1435 # 4. Destroying... 

1436 controller_is_running = self._controller_is_running() 

1437 uri = 'linstor://localhost' 

1438 try: 

1439 if controller_is_running: 

1440 self._start_controller(start=False) 

1441 

1442 # 4.1. Umount LINSTOR database. 

1443 self._mount_database_volume( 

1444 self.build_device_path(DATABASE_VOLUME_NAME), 

1445 mount=False, 

1446 force=True 

1447 ) 

1448 

1449 # 4.2. Refresh instance. 

1450 self._start_controller(start=True) 

1451 self._linstor = self._create_linstor_instance( 

1452 uri, keep_uri_unmodified=True 

1453 ) 

1454 

1455 # 4.3. Destroy database volume. 

1456 self._destroy_resource(DATABASE_VOLUME_NAME) 

1457 

1458 # 4.4. Refresh linstor connection. 

1459 # Without we get this error: 

1460 # "Cannot delete resource group 'xcp-sr-linstor_group_thin_device' because it has existing resource definitions.." 

1461 # Because the deletion of the databse was not seen by Linstor for some reason. 

1462 # It seems a simple refresh of the Linstor connection make it aware of the deletion. 

1463 self._linstor.disconnect() 

1464 self._linstor.connect() 

1465 

1466 # 4.5. Destroy remaining drbd nodes on hosts. 

1467 # We check if there is a DRBD node on hosts that could mean blocking when destroying resource groups. 

1468 # It needs to be done locally by each host so we go through the linstor-manager plugin. 

1469 # If we don't do this sometimes, the destroy will fail when trying to destroy the resource groups with: 

1470 # "linstor-manager:destroy error: Failed to destroy SP `xcp-sr-linstor_group_thin_device` on node `r620-s2`: The specified storage pool 'xcp-sr-linstor_group_thin_device' on node 'r620-s2' can not be deleted as volumes / snapshot-volumes are still using it." 

1471 session = util.timeout(5, util.ApiSession, "SM-linstor-destroy").session 

1472 for host_ref in session.xenapi.host.get_all(): 

1473 try: 

1474 response = session.xenapi.host.call_plugin( 

1475 host_ref, 'linstor-manager', 'destroyDrbdVolumes', {'volume_group': self._group_name} 

1476 ) 

1477 except Exception as e: 

1478 util.SMlog('Calling destroyDrbdVolumes on host {} failed with error {}'.format(host_ref, e)) 

1479 

1480 # 4.6. Destroy group and storage pools. 

1481 self._destroy_resource_group(self._linstor, self._group_name) 

1482 self._destroy_resource_group(self._linstor, self._ha_group_name) 

1483 for pool in self._get_storage_pools(force=True): 

1484 self._destroy_storage_pool( 

1485 self._linstor, pool.name, pool.node_name 

1486 ) 

1487 except Exception as e: 

1488 self._start_controller(start=controller_is_running) 

1489 raise e 

1490 

1491 try: 

1492 self._start_controller(start=False) 

1493 for file in os.listdir(DATABASE_PATH): 

1494 if file != 'lost+found': 

1495 os.remove(DATABASE_PATH + '/' + file) 

1496 except Exception as e: 

1497 util.SMlog( 

1498 'Ignoring failure after LINSTOR SR destruction: {}' 

1499 .format(e) 

1500 ) 

1501 

1502 def find_up_to_date_diskful_nodes(self, volume_uuid): 

1503 """ 

1504 Find all nodes that contain a specific volume using diskful disks. 

1505 The disk must be up to data to be used. 

1506 :param str volume_uuid: The volume to use. 

1507 :return: The available nodes. 

1508 :rtype: tuple(set(str), str) 

1509 """ 

1510 

1511 volume_name = self.get_volume_name(volume_uuid) 

1512 

1513 in_use_by = None 

1514 node_names = set() 

1515 

1516 resource_states = filter( 

1517 lambda resource_state: resource_state.name == volume_name, 

1518 self._get_resource_cache().resource_states 

1519 ) 

1520 

1521 for resource_state in resource_states: 

1522 volume_state = resource_state.volume_states[0] 

1523 if volume_state.disk_state == 'UpToDate': 

1524 node_names.add(resource_state.node_name) 

1525 if resource_state.in_use: 

1526 in_use_by = resource_state.node_name 

1527 

1528 return (node_names, in_use_by) 

1529 

1530 def invalidate_resource_cache(self): 

1531 """ 

1532 If resources are impacted by external commands like vhdutil, 

1533 it's necessary to call this function to invalidate current resource 

1534 cache. 

1535 """ 

1536 self._mark_resource_cache_as_dirty() 

1537 

1538 def has_node(self, node_name): 

1539 """ 

1540 Check if a node exists in the LINSTOR database. 

1541 :rtype: bool 

1542 """ 

1543 result = self._linstor.node_list() 

1544 error_str = self._get_error_str(result) 

1545 if error_str: 

1546 raise LinstorVolumeManagerError( 

1547 'Failed to list nodes using `{}`: {}' 

1548 .format(node_name, error_str) 

1549 ) 

1550 return bool(result[0].node(node_name)) 

1551 

1552 def create_node(self, node_name, ip): 

1553 """ 

1554 Create a new node in the LINSTOR database. 

1555 :param str node_name: Node name to use. 

1556 :param str ip: Host IP to communicate. 

1557 """ 

1558 result = self._linstor.node_create( 

1559 node_name, 

1560 linstor.consts.VAL_NODE_TYPE_CMBD, 

1561 ip 

1562 ) 

1563 errors = self._filter_errors(result) 

1564 if errors: 

1565 error_str = self._get_error_str(errors) 

1566 raise LinstorVolumeManagerError( 

1567 'Failed to create node `{}`: {}'.format(node_name, error_str) 

1568 ) 

1569 

1570 def destroy_node(self, node_name): 

1571 """ 

1572 Destroy a node in the LINSTOR database. 

1573 :param str node_name: Node name to remove. 

1574 """ 

1575 result = self._linstor.node_delete(node_name) 

1576 errors = self._filter_errors(result) 

1577 if errors: 

1578 error_str = self._get_error_str(errors) 

1579 raise LinstorVolumeManagerError( 

1580 'Failed to destroy node `{}`: {}'.format(node_name, error_str) 

1581 ) 

1582 

1583 def create_node_interface(self, node_name, name, ip): 

1584 """ 

1585 Create a new node interface in the LINSTOR database. 

1586 :param str node_name: Node name of the interface to use. 

1587 :param str name: Interface to create. 

1588 :param str ip: IP of the interface. 

1589 """ 

1590 result = self._linstor.netinterface_create(node_name, name, ip) 

1591 errors = self._filter_errors(result) 

1592 if errors: 

1593 error_str = self._get_error_str(errors) 

1594 raise LinstorVolumeManagerError( 

1595 'Failed to create node interface on `{}`: {}'.format(node_name, error_str) 

1596 ) 

1597 

1598 def destroy_node_interface(self, node_name, name): 

1599 """ 

1600 Destroy a node interface in the LINSTOR database. 

1601 :param str node_name: Node name of the interface to remove. 

1602 :param str name: Interface to remove. 

1603 """ 

1604 

1605 if name == 'default': 

1606 raise LinstorVolumeManagerError( 

1607 'Unable to delete the default interface of a node!' 

1608 ) 

1609 

1610 result = self._linstor.netinterface_delete(node_name, name) 

1611 errors = self._filter_errors(result) 

1612 if errors: 

1613 error_str = self._get_error_str(errors) 

1614 raise LinstorVolumeManagerError( 

1615 'Failed to destroy node interface on `{}`: {}'.format(node_name, error_str) 

1616 ) 

1617 

1618 def modify_node_interface(self, node_name, name, ip): 

1619 """ 

1620 Modify a node interface in the LINSTOR database. Create it if necessary. 

1621 :param str node_name: Node name of the interface to use. 

1622 :param str name: Interface to modify or create. 

1623 :param str ip: IP of the interface. 

1624 """ 

1625 result = self._linstor.netinterface_create(node_name, name, ip) 

1626 errors = self._filter_errors(result) 

1627 if not errors: 

1628 return 

1629 

1630 if self._check_errors(errors, [linstor.consts.FAIL_EXISTS_NET_IF]): 

1631 result = self._linstor.netinterface_modify(node_name, name, ip) 

1632 errors = self._filter_errors(result) 

1633 if not errors: 

1634 return 

1635 

1636 error_str = self._get_error_str(errors) 

1637 raise LinstorVolumeManagerError( 

1638 'Unable to modify interface on `{}`: {}'.format(node_name, error_str) 

1639 ) 

1640 

1641 def list_node_interfaces(self, node_name): 

1642 """ 

1643 List all node interfaces. 

1644 :param str node_name: Node name to use to list interfaces. 

1645 :rtype: list 

1646 : 

1647 """ 

1648 result = self._linstor.net_interface_list(node_name) 

1649 if not result: 

1650 raise LinstorVolumeManagerError( 

1651 'Unable to list interfaces on `{}`: no list received'.format(node_name) 

1652 ) 

1653 

1654 interfaces = {} 

1655 for interface in result: 

1656 interface = interface._rest_data 

1657 interfaces[interface['name']] = { 

1658 'address': interface['address'], 

1659 'active': interface['is_active'] 

1660 } 

1661 return interfaces 

1662 

1663 def get_node_preferred_interface(self, node_name): 

1664 """ 

1665 Get the preferred interface used by a node. 

1666 :param str node_name: Node name of the interface to get. 

1667 :rtype: str 

1668 """ 

1669 try: 

1670 nodes = self._linstor.node_list_raise([node_name]).nodes 

1671 if nodes: 

1672 properties = nodes[0].props 

1673 return properties.get('PrefNic', 'default') 

1674 raise LinstorVolumeManagerError('Node `{}` not found'.format(node_name)) 

1675 except Exception as e: 

1676 raise LinstorVolumeManagerError( 

1677 'Failed to get preferred interface: `{}`'.format(e) 

1678 ) 

1679 

1680 def set_node_preferred_interface(self, node_name, name): 

1681 """ 

1682 Set the preferred interface to use on a node. 

1683 :param str node_name: Node name of the interface. 

1684 :param str name: Preferred interface to use. 

1685 """ 

1686 result = self._linstor.node_modify(node_name, property_dict={'PrefNic': name}) 

1687 errors = self._filter_errors(result) 

1688 if errors: 

1689 error_str = self._get_error_str(errors) 

1690 raise LinstorVolumeManagerError( 

1691 'Failed to set preferred node interface on `{}`: {}'.format(node_name, error_str) 

1692 ) 

1693 

1694 def get_nodes_info(self): 

1695 """ 

1696 Get all nodes + statuses, used or not by the pool. 

1697 :rtype: dict(str, dict) 

1698 """ 

1699 try: 

1700 nodes = {} 

1701 for node in self._linstor.node_list_raise().nodes: 

1702 nodes[node.name] = node.connection_status 

1703 return nodes 

1704 except Exception as e: 

1705 raise LinstorVolumeManagerError( 

1706 'Failed to get all nodes: `{}`'.format(e) 

1707 ) 

1708 

1709 def get_storage_pools_info(self): 

1710 """ 

1711 Give all storage pools of current group name. 

1712 :rtype: dict(str, list) 

1713 """ 

1714 storage_pools = {} 

1715 for pool in self._get_storage_pools(force=True): 

1716 if pool.node_name not in storage_pools: 

1717 storage_pools[pool.node_name] = [] 

1718 

1719 size = -1 

1720 capacity = -1 

1721 

1722 space = pool.free_space 

1723 if space: 

1724 size = space.free_capacity 

1725 if size < 0: 

1726 size = -1 

1727 else: 

1728 size *= 1024 

1729 capacity = space.total_capacity 

1730 if capacity <= 0: 

1731 capacity = -1 

1732 else: 

1733 capacity *= 1024 

1734 

1735 storage_pools[pool.node_name].append({ 

1736 'name': pool.name, 

1737 'linstor-uuid': pool.uuid, 

1738 'free-size': size, 

1739 'capacity': capacity 

1740 }) 

1741 

1742 return storage_pools 

1743 

1744 def get_resources_info(self): 

1745 """ 

1746 Give all resources of current group name. 

1747 :rtype: dict(str, list) 

1748 """ 

1749 if self._resources_info_cache and not self._resource_cache_dirty: 

1750 return self._resources_info_cache 

1751 

1752 resources = {} 

1753 resource_list = self._get_resource_cache() 

1754 volume_names = self.get_volumes_with_name() 

1755 for resource in resource_list.resources: 

1756 if resource.name not in resources: 

1757 resources[resource.name] = { 'nodes': {}, 'uuid': '' } 

1758 resource_nodes = resources[resource.name]['nodes'] 

1759 

1760 resource_nodes[resource.node_name] = { 

1761 'volumes': [], 

1762 'diskful': linstor.consts.FLAG_DISKLESS not in resource.flags, 

1763 'tie-breaker': linstor.consts.FLAG_TIE_BREAKER in resource.flags 

1764 } 

1765 resource_volumes = resource_nodes[resource.node_name]['volumes'] 

1766 

1767 for volume in resource.volumes: 

1768 # We ignore diskless pools of the form "DfltDisklessStorPool". 

1769 if volume.storage_pool_name != self._group_name: 

1770 continue 

1771 

1772 usable_size = volume.usable_size 

1773 if usable_size < 0: 

1774 usable_size = -1 

1775 else: 

1776 usable_size *= 1024 

1777 

1778 allocated_size = volume.allocated_size 

1779 if allocated_size < 0: 

1780 allocated_size = -1 

1781 else: 

1782 allocated_size *= 1024 

1783 

1784 resource_volumes.append({ 

1785 'storage-pool-name': volume.storage_pool_name, 

1786 'linstor-uuid': volume.uuid, 

1787 'number': volume.number, 

1788 'device-path': volume.device_path, 

1789 'usable-size': usable_size, 

1790 'allocated-size': allocated_size 

1791 }) 

1792 

1793 for resource_state in resource_list.resource_states: 

1794 resource = resources[resource_state.rsc_name]['nodes'][resource_state.node_name] 

1795 resource['in-use'] = resource_state.in_use 

1796 

1797 volumes = resource['volumes'] 

1798 for volume_state in resource_state.volume_states: 

1799 volume = next((x for x in volumes if x['number'] == volume_state.number), None) 

1800 if volume: 

1801 volume['disk-state'] = volume_state.disk_state 

1802 

1803 for volume_uuid, volume_name in volume_names.items(): 

1804 resource = resources.get(volume_name) 

1805 if resource: 

1806 resource['uuid'] = volume_uuid 

1807 

1808 self._resources_info_cache = resources 

1809 return self._resources_info_cache 

1810 

1811 def get_resource_info(self, volume_uuid: str) -> Dict[str, Any]: 

1812 """ 

1813 Give a resource info based on its UUID. 

1814 :param volume_uuid str: volume uuid to search for 

1815 :rtype: dict(str, any) 

1816 """ 

1817 for volume in self.get_resources_info().values(): 

1818 if volume["uuid"] == volume_uuid: 

1819 return volume 

1820 

1821 raise LinstorVolumeManagerError( 

1822 f"Could not find info about volume `{volume_uuid}`", 

1823 LinstorVolumeManagerError.ERR_VOLUME_NOT_EXISTS 

1824 ) 

1825 

1826 def get_database_path(self): 

1827 """ 

1828 Get the database path. 

1829 :return: The current database path. 

1830 :rtype: str 

1831 """ 

1832 return self._request_database_path(self._linstor, activate=True) 

1833 

1834 @classmethod 

1835 def is_controller(cls): 

1836 return cls._is_mounted(DATABASE_PATH) 

1837 

1838 @classmethod 

1839 def get_database_backup_age(cls): 

1840 """ 

1841 Return the latest backup age in seconds. 

1842 If not called on the Controller, since backups are not available, 

1843 returns a huge value (a timestamp of now). 

1844 """ 

1845 return (datetime.now() - cls._get_latest_database_backup()[1]).total_seconds() 

1846 

1847 def database_backup(self, name=""): 

1848 # Create new backup 

1849 date = datetime.now().strftime(DATABASE_BACKUP_DATE_FORMAT) 

1850 filename = DATABASE_BACKUP_NAME_FORMAT.format(date, name) 

1851 self._linstor.controller_backupdb(filename) 

1852 # Relative path are ok for a secondary backup filename: 

1853 # https://github.com/LINBIT/linstor-server/blob/3e9306a9d8215606544c64c50ced150625ee4926/controller/src/main/java/com/linbit/linstor/api/rest/v1/Controller.java#L408 

1854 self._linstor.controller_backupdb(f"../linstor.d/db-backups/{filename}") 

1855 util.SMlog(f"[database_backup] Created: {filename}", priority=util.LOG_INFO) 

1856 

1857 @classmethod 

1858 def database_backup_validate_and_prune(cls): 

1859 """ 

1860 Removes old backup based on two criterias: 

1861 - Validity of the zipfile done by self._check_database_backup. 

1862 - Number of valid files found, only the nth latest are kept. 

1863 """ 

1864 for directory in (DATABASE_BACKUP_DIR_MAIN, DATABASE_BACKUP_DIR_SPARE): 

1865 valid_backup_count = 0 

1866 # Validate file and apply retention 

1867 for database_backup_path, _ in cls._get_sorted_database_backup(directory): 

1868 try: 

1869 cls._check_database_backup(database_backup_path) 

1870 valid_backup_count += 1 

1871 if valid_backup_count < DATABASE_BACKUP_RETENTION: 

1872 continue 

1873 except LinstorDatabaseBackupError as error: 

1874 util.SMlog(f"[database_backup] Check failed `{error}` [{database_backup_path}]", 

1875 priority=util.LOG_ERR) 

1876 with contextlib.suppress(OSError): 

1877 os.unlink(database_backup_path) 

1878 

1879 @classmethod 

1880 def get_all_group_names(cls, base_name): 

1881 """ 

1882 Get all group names. I.e. list of current group + HA. 

1883 :param str base_name: The SR group_name to use. 

1884 :return: List of group names. 

1885 :rtype: list 

1886 """ 

1887 return [cls.build_group_name(base_name), cls._build_ha_group_name(base_name)] 

1888 

1889 @classmethod 

1890 def get_volume_group_name(cls, volume_name) -> str: 

1891 """ 

1892 Get the group name associated with a volume. 

1893 :param str volume_name: The volume name to find. 

1894 :return: A group name. 

1895 :rtype: str 

1896 """ 

1897 

1898 lin = cls._create_linstor_instance(uri=None) 

1899 try: 

1900 for dfn in lin.resource_dfn_list_raise( 

1901 query_volume_definitions=False, 

1902 filter_by_resource_definitions=[volume_name] 

1903 ).resource_definitions: 

1904 return dfn.resource_group_name 

1905 finally: 

1906 lin.disconnect() 

1907 

1908 return '' 

1909 

1910 @classmethod 

1911 def create_sr(cls, group_name, ips, redundancy, thin_provisioning, logger=default_logger.__func__): 

1912 """ 

1913 Create a new SR on the given nodes. 

1914 :param str group_name: The SR group_name to use. 

1915 :param set(str) ips: Node ips. 

1916 :param int redundancy: How many copy of volumes should we store? 

1917 :param bool thin_provisioning: Use thin or thick provisioning. 

1918 :param function logger: Function to log messages. 

1919 :return: A new LinstorSr instance. 

1920 :rtype: LinstorSr 

1921 """ 

1922 

1923 try: 

1924 cls._start_controller(start=True) 

1925 sr = cls._create_sr(group_name, ips, redundancy, thin_provisioning, logger) 

1926 finally: 

1927 # Controller must be stopped and volume unmounted because 

1928 # it is the role of the drbd-reactor daemon to do the right 

1929 # actions. 

1930 cls._start_controller(start=False) 

1931 cls._mount_volume( 

1932 cls.build_device_path(DATABASE_VOLUME_NAME), 

1933 DATABASE_PATH, 

1934 mount=False 

1935 ) 

1936 return sr 

1937 

1938 @classmethod 

1939 def _create_sr(cls, group_name, ips, redundancy, thin_provisioning, logger=default_logger.__func__): 

1940 # 1. Check if SR already exists. 

1941 uri = 'linstor://localhost' 

1942 

1943 lin = cls._create_linstor_instance(uri, keep_uri_unmodified=True) 

1944 

1945 node_names = list(ips.keys()) 

1946 for node_name, ip in ips.items(): 

1947 while True: 

1948 # Try to create node. 

1949 result = lin.node_create( 

1950 node_name, 

1951 linstor.consts.VAL_NODE_TYPE_CMBD, 

1952 ip 

1953 ) 

1954 

1955 errors = cls._filter_errors(result) 

1956 if cls._check_errors( 

1957 errors, [linstor.consts.FAIL_EXISTS_NODE] 

1958 ): 

1959 # If it already exists, remove, then recreate. 

1960 result = lin.node_delete(node_name) 

1961 error_str = cls._get_error_str(result) 

1962 if error_str: 

1963 raise LinstorVolumeManagerError( 

1964 'Failed to remove old node `{}`: {}' 

1965 .format(node_name, error_str) 

1966 ) 

1967 elif not errors: 

1968 break # Created! 

1969 else: 

1970 raise LinstorVolumeManagerError( 

1971 'Failed to create node `{}` with ip `{}`: {}'.format( 

1972 node_name, ip, cls._get_error_str(errors) 

1973 ) 

1974 ) 

1975 

1976 driver_pool_name = group_name 

1977 base_group_name = group_name 

1978 group_name = cls.build_group_name(group_name) 

1979 storage_pool_name = group_name 

1980 pools = lin.storage_pool_list_raise(filter_by_stor_pools=[storage_pool_name]).storage_pools 

1981 if pools: 

1982 existing_node_names = [pool.node_name for pool in pools] 

1983 raise LinstorVolumeManagerError( 

1984 'Unable to create SR `{}`. It already exists on node(s): {}' 

1985 .format(group_name, existing_node_names) 

1986 ) 

1987 

1988 if lin.resource_group_list_raise( 

1989 cls.get_all_group_names(base_group_name) 

1990 ).resource_groups: 

1991 if not lin.resource_dfn_list_raise().resource_definitions: 

1992 backup_path = cls._create_database_backup_path() 

1993 logger( 

1994 'Group name already exists `{}` without LVs. ' 

1995 'Ignoring and moving the config files in {}'.format(group_name, backup_path) 

1996 ) 

1997 cls._move_files(DATABASE_PATH, backup_path) 

1998 else: 

1999 raise LinstorVolumeManagerError( 

2000 'Unable to create SR `{}`: The group name already exists' 

2001 .format(group_name) 

2002 ) 

2003 

2004 if thin_provisioning: 

2005 driver_pool_parts = driver_pool_name.split('/') 

2006 if not len(driver_pool_parts) == 2: 

2007 raise LinstorVolumeManagerError( 

2008 'Invalid group name using thin provisioning. ' 

2009 'Expected format: \'VG/LV`\'' 

2010 ) 

2011 

2012 # 2. Create storage pool on each node + resource group. 

2013 reg_volume_group_not_found = re.compile( 

2014 ".*Volume group '.*' not found$" 

2015 ) 

2016 

2017 i = 0 

2018 try: 

2019 # 2.a. Create storage pools. 

2020 storage_pool_count = 0 

2021 while i < len(node_names): 

2022 node_name = node_names[i] 

2023 

2024 result = lin.storage_pool_create( 

2025 node_name=node_name, 

2026 storage_pool_name=storage_pool_name, 

2027 storage_driver='LVM_THIN' if thin_provisioning else 'LVM', 

2028 driver_pool_name=driver_pool_name 

2029 ) 

2030 

2031 errors = linstor.Linstor.filter_api_call_response_errors( 

2032 result 

2033 ) 

2034 if errors: 

2035 if errors[0].is_error( 

2036 linstor.consts.FAIL_STOR_POOL_CONFIGURATION_ERROR 

2037 ) and reg_volume_group_not_found.match(errors[0].message): 

2038 logger( 

2039 'Volume group `{}` not found on `{}`. Ignoring...' 

2040 .format(group_name, node_name) 

2041 ) 

2042 cls._destroy_storage_pool(lin, storage_pool_name, node_name) 

2043 else: 

2044 error_str = cls._get_error_str(result) 

2045 raise LinstorVolumeManagerError( 

2046 'Could not create SP `{}` on node `{}`: {}' 

2047 .format(group_name, node_name, error_str) 

2048 ) 

2049 else: 

2050 storage_pool_count += 1 

2051 i += 1 

2052 

2053 if not storage_pool_count: 

2054 raise LinstorVolumeManagerError( 

2055 'Unable to create SR `{}`: No VG group found'.format( 

2056 group_name, 

2057 ) 

2058 ) 

2059 

2060 # 2.b. Create resource groups. 

2061 ha_group_name = cls._build_ha_group_name(base_group_name) 

2062 cls._create_resource_group( 

2063 lin, 

2064 group_name, 

2065 storage_pool_name, 

2066 redundancy, 

2067 True 

2068 ) 

2069 cls._create_resource_group( 

2070 lin, 

2071 ha_group_name, 

2072 storage_pool_name, 

2073 3, 

2074 True 

2075 ) 

2076 

2077 # 3. Create the LINSTOR database volume and mount it. 

2078 try: 

2079 logger('Creating database volume...') 

2080 volume_path = cls._create_database_volume( 

2081 lin, ha_group_name, storage_pool_name, node_names, redundancy 

2082 ) 

2083 except LinstorVolumeManagerError as e: 

2084 if e.code != LinstorVolumeManagerError.ERR_VOLUME_EXISTS: 

2085 logger('Destroying database volume after creation fail...') 

2086 cls._force_destroy_database_volume(lin, group_name) 

2087 raise 

2088 

2089 try: 

2090 logger('Mounting database volume...') 

2091 

2092 # First we must disable the controller to move safely the 

2093 # LINSTOR config. 

2094 cls._start_controller(start=False) 

2095 

2096 cls._mount_database_volume(volume_path) 

2097 except Exception as e: 

2098 # Ensure we are connected because controller has been 

2099 # restarted during mount call. 

2100 logger('Destroying database volume after mount fail...') 

2101 

2102 try: 

2103 cls._start_controller(start=True) 

2104 except Exception: 

2105 pass 

2106 

2107 lin = cls._create_linstor_instance( 

2108 uri, keep_uri_unmodified=True 

2109 ) 

2110 cls._force_destroy_database_volume(lin, group_name) 

2111 raise e 

2112 

2113 cls._start_controller(start=True) 

2114 lin = cls._create_linstor_instance(uri, keep_uri_unmodified=True) 

2115 

2116 # 4. Remove storage pools/resource/volume group in the case of errors. 

2117 except Exception as e: 

2118 logger('Destroying resource group and storage pools after fail...') 

2119 try: 

2120 cls._destroy_resource_group(lin, group_name) 

2121 cls._destroy_resource_group(lin, ha_group_name) 

2122 except Exception as e2: 

2123 logger('Failed to destroy resource group: {}'.format(e2)) 

2124 pass 

2125 j = 0 

2126 i = min(i, len(node_names) - 1) 

2127 while j <= i: 

2128 try: 

2129 cls._destroy_storage_pool(lin, storage_pool_name, node_names[j]) 

2130 except Exception as e2: 

2131 logger('Failed to destroy resource group: {}'.format(e2)) 

2132 pass 

2133 j += 1 

2134 raise e 

2135 

2136 # 5. Return new instance. 

2137 instance = cls.__new__(cls) 

2138 instance._linstor = lin 

2139 instance._logger = logger 

2140 instance._redundancy = redundancy 

2141 instance._base_group_name = base_group_name 

2142 instance._group_name = group_name 

2143 instance._volumes = set() 

2144 instance._storage_pools_time = 0 

2145 instance._kv_cache = instance._create_kv_cache() 

2146 instance._resource_cache = None 

2147 instance._resource_cache_dirty = True 

2148 instance._volume_info_cache = None 

2149 instance._volume_info_cache_dirty = True 

2150 return instance 

2151 

2152 @classmethod 

2153 def build_device_path(cls, volume_name): 

2154 """ 

2155 Build a device path given a volume name. 

2156 :param str volume_name: The volume name to use. 

2157 :return: A valid or not device path. 

2158 :rtype: str 

2159 """ 

2160 

2161 return '{}{}/0'.format(cls.DEV_ROOT_PATH, volume_name) 

2162 

2163 @classmethod 

2164 def build_volume_name(cls, base_name): 

2165 """ 

2166 Build a volume name given a base name (i.e. a UUID). 

2167 :param str base_name: The volume name to use. 

2168 :return: A valid or not device path. 

2169 :rtype: str 

2170 """ 

2171 return '{}{}'.format(cls.PREFIX_VOLUME, base_name) 

2172 

2173 @classmethod 

2174 def round_up_volume_size(cls, volume_size): 

2175 """ 

2176 Align volume size on higher multiple of BLOCK_SIZE. 

2177 :param int volume_size: The volume size to align. 

2178 :return: An aligned volume size. 

2179 :rtype: int 

2180 """ 

2181 return round_up(volume_size, cls.BLOCK_SIZE) 

2182 

2183 @classmethod 

2184 def round_down_volume_size(cls, volume_size): 

2185 """ 

2186 Align volume size on lower multiple of BLOCK_SIZE. 

2187 :param int volume_size: The volume size to align. 

2188 :return: An aligned volume size. 

2189 :rtype: int 

2190 """ 

2191 return round_down(volume_size, cls.BLOCK_SIZE) 

2192 

2193 # -------------------------------------------------------------------------- 

2194 # Private helpers. 

2195 # -------------------------------------------------------------------------- 

2196 

2197 def _create_kv_cache(self): 

2198 self._kv_cache = self._create_linstor_kv('/') 

2199 self._kv_cache_dirty = False 

2200 return self._kv_cache 

2201 

2202 def _get_kv_cache(self): 

2203 if self._kv_cache_dirty: 

2204 self._kv_cache = self._create_kv_cache() 

2205 return self._kv_cache 

2206 

2207 def _create_resource_cache(self): 

2208 self._resource_cache = self._linstor.resource_list_raise() 

2209 self._resource_cache_dirty = False 

2210 return self._resource_cache 

2211 

2212 def _get_resource_cache(self): 

2213 if self._resource_cache_dirty: 

2214 self._resource_cache = self._create_resource_cache() 

2215 return self._resource_cache 

2216 

2217 def _mark_resource_cache_as_dirty(self): 

2218 self._resource_cache_dirty = True 

2219 self._volume_info_cache_dirty = True 

2220 

2221 # -------------------------------------------------------------------------- 

2222 

2223 def _ensure_volume_exists(self, volume_uuid): 

2224 if volume_uuid not in self._volumes: 

2225 raise LinstorVolumeManagerError( 

2226 'volume `{}` doesn\'t exist'.format(volume_uuid), 

2227 LinstorVolumeManagerError.ERR_VOLUME_NOT_EXISTS 

2228 ) 

2229 

2230 def _find_best_size_candidates(self): 

2231 result = self._linstor.resource_group_qmvs(self._group_name) 

2232 error_str = self._get_error_str(result) 

2233 if error_str: 

2234 raise LinstorVolumeManagerError( 

2235 'Failed to get max volume size allowed of SR `{}`: {}'.format( 

2236 self._group_name, 

2237 error_str 

2238 ) 

2239 ) 

2240 return result[0].candidates 

2241 

2242 def _fetch_resource_names(self, ignore_deleted=True): 

2243 resource_names = set() 

2244 dfns = self._linstor.resource_dfn_list_raise().resource_definitions 

2245 for dfn in dfns: 

2246 if dfn.resource_group_name in self.get_all_group_names(self._base_group_name) and ( 

2247 ignore_deleted or 

2248 linstor.consts.FLAG_DELETE not in dfn.flags 

2249 ): 

2250 resource_names.add(dfn.name) 

2251 return resource_names 

2252 

2253 def _get_volumes_info(self, volume_names=None): 

2254 all_volume_info = {} 

2255 

2256 if not self._volume_info_cache_dirty: 

2257 return self._volume_info_cache 

2258 

2259 # `volume_names` MUST contain all volumes registered in the KV store. 

2260 # It can be provided to the function to avoid double fetching. 

2261 if not volume_names: 

2262 volume_names = self.get_volumes_with_name() 

2263 volume_names = set(volume_names.values()) 

2264 

2265 def process_resource(resource): 

2266 if resource.name not in all_volume_info: 

2267 current = all_volume_info[resource.name] = self.VolumeInfo( 

2268 resource.name 

2269 ) 

2270 else: 

2271 current = all_volume_info[resource.name] 

2272 

2273 if linstor.consts.FLAG_DISKLESS not in resource.flags: 

2274 current.diskful.append(resource.node_name) 

2275 

2276 for volume in resource.volumes: 

2277 # We ignore diskless pools of the form "DfltDisklessStorPool". 

2278 if volume.storage_pool_name != self._group_name: 

2279 continue 

2280 # Only fetch first volume. 

2281 if volume.number != 0: 

2282 continue 

2283 

2284 allocated_size = volume.allocated_size 

2285 if allocated_size > current.allocated_size: 

2286 current.allocated_size = allocated_size 

2287 

2288 usable_size = volume.usable_size 

2289 if usable_size > 0 and ( 

2290 usable_size < current.virtual_size or 

2291 not current.virtual_size 

2292 ): 

2293 current.virtual_size = usable_size 

2294 

2295 try: 

2296 for resource in self._get_resource_cache().resources: 

2297 if resource.name in volume_names: 

2298 process_resource(resource) 

2299 for volume in all_volume_info.values(): 

2300 if volume.allocated_size <= 0: 

2301 raise LinstorVolumeManagerError('Failed to get allocated size of `{}`'.format(resource.name)) 

2302 

2303 if volume.virtual_size <= 0: 

2304 raise LinstorVolumeManagerError('Failed to get usable size of `{}`'.format(volume.name)) 

2305 

2306 volume.allocated_size *= 1024 

2307 volume.virtual_size *= 1024 

2308 except LinstorVolumeManagerError: 

2309 self._mark_resource_cache_as_dirty() 

2310 raise 

2311 

2312 self._volume_info_cache_dirty = False 

2313 self._volume_info_cache = all_volume_info 

2314 

2315 return all_volume_info 

2316 

2317 def _get_volume_node_names_and_size(self, volume_name): 

2318 node_names = set() 

2319 size = -1 

2320 for resource in self._linstor.resource_list_raise( 

2321 filter_by_resources=[volume_name] 

2322 ).resources: 

2323 for volume in resource.volumes: 

2324 # We ignore diskless pools of the form "DfltDisklessStorPool". 

2325 if volume.storage_pool_name != self._group_name: 

2326 continue 

2327 

2328 node_names.add(resource.node_name) 

2329 

2330 usable_size = volume.usable_size 

2331 if usable_size <= 0: 

2332 continue 

2333 

2334 if size < 0: 

2335 size = usable_size 

2336 else: 

2337 size = min(size, usable_size) 

2338 

2339 if size <= 0: 

2340 raise LinstorVolumeManagerError('Failed to get usable size of `{}`'.format(resource.name)) 

2341 

2342 return (node_names, size * 1024) 

2343 

2344 def _compute_size(self, attr): 

2345 capacity = 0 

2346 for pool in self._get_storage_pools(force=True): 

2347 space = pool.free_space 

2348 if space: 

2349 size = getattr(space, attr) 

2350 if size < 0: 

2351 raise LinstorVolumeManagerError( 

2352 'Failed to get pool {} attr of `{}`' 

2353 .format(attr, pool.node_name) 

2354 ) 

2355 capacity += size 

2356 return capacity * 1024 

2357 

2358 def _get_node_names(self): 

2359 node_names = set() 

2360 for pool in self._get_storage_pools(): 

2361 node_names.add(pool.node_name) 

2362 return node_names 

2363 

2364 def _get_storage_pools(self, force=False): 

2365 cur_time = time.time() 

2366 elsaped_time = cur_time - self._storage_pools_time 

2367 

2368 if force or elsaped_time >= self.STORAGE_POOLS_FETCH_INTERVAL: 

2369 self._storage_pools = self._linstor.storage_pool_list_raise( 

2370 filter_by_stor_pools=[self._group_name] 

2371 ).storage_pools 

2372 self._storage_pools_time = time.time() 

2373 

2374 return self._storage_pools 

2375 

2376 def _create_volume( 

2377 self, 

2378 volume_uuid, 

2379 volume_name, 

2380 size, 

2381 place_resources, 

2382 high_availability 

2383 ): 

2384 size = self.round_up_volume_size(size) 

2385 self._mark_resource_cache_as_dirty() 

2386 

2387 group_name = self._ha_group_name if high_availability else self._group_name 

2388 def create_definition(): 

2389 first_attempt = True 

2390 while True: 

2391 try: 

2392 self._check_volume_creation_errors( 

2393 self._linstor.resource_group_spawn( 

2394 rsc_grp_name=group_name, 

2395 rsc_dfn_name=volume_name, 

2396 vlm_sizes=['{}B'.format(size)], 

2397 definitions_only=True 

2398 ), 

2399 volume_uuid, 

2400 self._group_name 

2401 ) 

2402 break 

2403 except LinstorVolumeManagerError as e: 

2404 if ( 

2405 not first_attempt or 

2406 not high_availability or 

2407 e.code != LinstorVolumeManagerError.ERR_GROUP_NOT_EXISTS 

2408 ): 

2409 raise 

2410 

2411 first_attempt = False 

2412 self._create_resource_group( 

2413 self._linstor, 

2414 group_name, 

2415 self._group_name, 

2416 3, 

2417 True 

2418 ) 

2419 

2420 self._configure_volume_peer_slots(self._linstor, volume_name) 

2421 

2422 def clean(): 

2423 try: 

2424 self._destroy_volume(volume_uuid, force=True, preserve_properties=True) 

2425 except Exception as e: 

2426 self._logger( 

2427 'Unable to destroy volume {} after creation fail: {}' 

2428 .format(volume_uuid, e) 

2429 ) 

2430 

2431 def create(): 

2432 try: 

2433 create_definition() 

2434 if place_resources: 

2435 # Basic case when we use the default redundancy of the group. 

2436 self._check_volume_creation_errors( 

2437 self._linstor.resource_auto_place( 

2438 rsc_name=volume_name, 

2439 place_count=self._redundancy, 

2440 diskless_on_remaining=False 

2441 ), 

2442 volume_uuid, 

2443 self._group_name 

2444 ) 

2445 except LinstorVolumeManagerError as e: 

2446 if e.code != LinstorVolumeManagerError.ERR_VOLUME_EXISTS: 

2447 clean() 

2448 raise 

2449 except Exception: 

2450 clean() 

2451 raise 

2452 

2453 util.retry(create, maxretry=5) 

2454 

2455 def _create_volume_with_properties( 

2456 self, 

2457 volume_uuid, 

2458 volume_name, 

2459 size, 

2460 place_resources, 

2461 high_availability 

2462 ): 

2463 if self.check_volume_exists(volume_uuid): 

2464 raise LinstorVolumeManagerError( 

2465 'Could not create volume `{}` from SR `{}`, it already exists' 

2466 .format(volume_uuid, self._group_name) + ' in properties', 

2467 LinstorVolumeManagerError.ERR_VOLUME_EXISTS 

2468 ) 

2469 

2470 if volume_name in self._fetch_resource_names(): 

2471 raise LinstorVolumeManagerError( 

2472 'Could not create volume `{}` from SR `{}`, '.format( 

2473 volume_uuid, self._group_name 

2474 ) + 'resource of the same name already exists in LINSTOR' 

2475 ) 

2476 

2477 # I am paranoid. 

2478 volume_properties = self._get_volume_properties(volume_uuid) 

2479 if (volume_properties.get(self.PROP_NOT_EXISTS) is not None): 

2480 raise LinstorVolumeManagerError( 

2481 'Could not create volume `{}`, '.format(volume_uuid) + 

2482 'properties already exist' 

2483 ) 

2484 

2485 try: 

2486 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_CREATING 

2487 volume_properties[self.PROP_VOLUME_NAME] = volume_name 

2488 

2489 self._create_volume( 

2490 volume_uuid, 

2491 volume_name, 

2492 size, 

2493 place_resources, 

2494 high_availability 

2495 ) 

2496 

2497 assert volume_properties.namespace == \ 

2498 self._build_volume_namespace(volume_uuid) 

2499 return volume_properties 

2500 except LinstorVolumeManagerError as e: 

2501 # Do not destroy existing resource! 

2502 # In theory we can't get this error because we check this event 

2503 # before the `self._create_volume` case. 

2504 # It can only happen if the same volume uuid is used in the same 

2505 # call in another host. 

2506 if e.code != LinstorVolumeManagerError.ERR_VOLUME_EXISTS: 

2507 self._destroy_volume(volume_uuid, force=True) 

2508 raise 

2509 

2510 def _find_device_path(self, volume_uuid, volume_name): 

2511 current_device_path = self._request_device_path( 

2512 volume_uuid, volume_name, activate=True 

2513 ) 

2514 

2515 # We use realpath here to get the /dev/drbd<id> path instead of 

2516 # /dev/drbd/by-res/<resource_name>. 

2517 expected_device_path = self.build_device_path(volume_name) 

2518 util.wait_for_path(expected_device_path, 5) 

2519 

2520 device_realpath = os.path.realpath(expected_device_path) 

2521 if current_device_path != device_realpath: 

2522 raise LinstorVolumeManagerError( 

2523 'Invalid path, current={}, expected={} (realpath={})' 

2524 .format( 

2525 current_device_path, 

2526 expected_device_path, 

2527 device_realpath 

2528 ) 

2529 ) 

2530 return expected_device_path 

2531 

2532 def _request_device_path(self, volume_uuid, volume_name, activate=False): 

2533 node_name = socket.gethostname() 

2534 

2535 resource = next(filter( 

2536 lambda resource: resource.node_name == node_name and 

2537 resource.name == volume_name, 

2538 self._get_resource_cache().resources 

2539 ), None) 

2540 

2541 if not resource: 

2542 if activate: 

2543 self._mark_resource_cache_as_dirty() 

2544 self._activate_device_path( 

2545 self._linstor, node_name, volume_name 

2546 ) 

2547 return self._request_device_path(volume_uuid, volume_name) 

2548 raise LinstorVolumeManagerError( 

2549 'Unable to get dev path for `{}`, no resource found but definition "seems" to exist' 

2550 .format(volume_uuid) 

2551 ) 

2552 

2553 # Contains a path of the /dev/drbd<id> form. 

2554 device_path = resource.volumes[0].device_path 

2555 if not device_path: 

2556 raise LinstorVolumeManagerError('Empty dev path for `{}`!'.format(volume_uuid)) 

2557 return device_path 

2558 

2559 def _destroy_resource(self, resource_name, force=False): 

2560 result = self._linstor.resource_dfn_delete(resource_name) 

2561 error_str = self._get_error_str(result) 

2562 if not error_str: 

2563 self._mark_resource_cache_as_dirty() 

2564 return 

2565 

2566 if not force: 

2567 self._mark_resource_cache_as_dirty() 

2568 raise LinstorVolumeManagerError( 

2569 'Could not destroy resource `{}` from SR `{}`: {}' 

2570 .format(resource_name, self._group_name, error_str) 

2571 ) 

2572 

2573 # If force is used, ensure there is no opener. 

2574 openers = get_all_volume_openers(resource_name, '0') 

2575 for host_openers in openers.values(): 

2576 if host_openers: 

2577 self._mark_resource_cache_as_dirty() 

2578 raise LinstorVolumeManagerError( 

2579 'Could not force destroy resource `{}` from SR `{}`: {} (openers=`{}`)' 

2580 .format(resource_name, self._group_name, error_str, openers) 

2581 ) 

2582 

2583 # Maybe the resource is blocked in primary mode. DRBD/LINSTOR issue? 

2584 resource_states = filter( 

2585 lambda resource_state: resource_state.name == resource_name, 

2586 self._get_resource_cache().resource_states 

2587 ) 

2588 

2589 # Mark only after computation of states. 

2590 self._mark_resource_cache_as_dirty() 

2591 

2592 for resource_state in resource_states: 

2593 volume_state = resource_state.volume_states[0] 

2594 if resource_state.in_use: 

2595 demote_drbd_resource(resource_state.node_name, resource_name) 

2596 break 

2597 self._destroy_resource(resource_name) 

2598 

2599 def _destroy_volume(self, volume_uuid, force=False, preserve_properties=False): 

2600 volume_properties = self._get_volume_properties(volume_uuid) 

2601 try: 

2602 volume_name = volume_properties.get(self.PROP_VOLUME_NAME) 

2603 if volume_name in self._fetch_resource_names(): 

2604 self._destroy_resource(volume_name, force) 

2605 

2606 # Assume this call is atomic. 

2607 if not preserve_properties: 

2608 volume_properties.clear() 

2609 except Exception as e: 

2610 raise LinstorVolumeManagerError( 

2611 'Cannot destroy volume `{}`: {}'.format(volume_uuid, e) 

2612 ) 

2613 

2614 def _build_volumes(self, repair): 

2615 properties = self._kv_cache 

2616 resource_names = self._fetch_resource_names() 

2617 

2618 self._volumes = set() 

2619 

2620 updating_uuid_volumes = self._get_volumes_by_property( 

2621 self.REG_UPDATING_UUID_SRC, ignore_inexisting_volumes=False 

2622 ) 

2623 if updating_uuid_volumes and not repair: 

2624 raise LinstorVolumeManagerError( 

2625 'Cannot build LINSTOR volume list: ' 

2626 'It exists invalid "updating uuid volumes", repair is required', 

2627 LinstorVolumeManagerError.ERR_REPAIR_REQUIRED 

2628 ) 

2629 

2630 existing_volumes = self._get_volumes_by_property( 

2631 self.REG_NOT_EXISTS, ignore_inexisting_volumes=False 

2632 ) 

2633 for volume_uuid, not_exists in existing_volumes.items(): 

2634 properties.namespace = self._build_volume_namespace(volume_uuid) 

2635 

2636 src_uuid = properties.get(self.PROP_UPDATING_UUID_SRC) 

2637 if src_uuid: 

2638 self._logger( 

2639 'Ignoring volume during manager initialization with prop ' 

2640 ' PROP_UPDATING_UUID_SRC: {} (properties={})' 

2641 .format( 

2642 volume_uuid, 

2643 self._get_filtered_properties(properties) 

2644 ) 

2645 ) 

2646 continue 

2647 

2648 # Insert volume in list if the volume exists. Or if the volume 

2649 # is being created and a slave wants to use it (repair = False). 

2650 # 

2651 # If we are on the master and if repair is True and state is 

2652 # Creating, it's probably a bug or crash: the creation process has 

2653 # been stopped. 

2654 if not_exists == self.STATE_EXISTS or ( 

2655 not repair and not_exists == self.STATE_CREATING 

2656 ): 

2657 self._volumes.add(volume_uuid) 

2658 continue 

2659 

2660 if not repair: 

2661 self._logger( 

2662 'Ignoring bad volume during manager initialization: {} ' 

2663 '(properties={})'.format( 

2664 volume_uuid, 

2665 self._get_filtered_properties(properties) 

2666 ) 

2667 ) 

2668 continue 

2669 

2670 # Remove bad volume. 

2671 try: 

2672 self._logger( 

2673 'Removing bad volume during manager initialization: {} ' 

2674 '(properties={})'.format( 

2675 volume_uuid, 

2676 self._get_filtered_properties(properties) 

2677 ) 

2678 ) 

2679 volume_name = properties.get(self.PROP_VOLUME_NAME) 

2680 

2681 # Little optimization, don't call `self._destroy_volume`, 

2682 # we already have resource name list. 

2683 if volume_name in resource_names: 

2684 self._destroy_resource(volume_name, force=True) 

2685 

2686 # Assume this call is atomic. 

2687 properties.clear() 

2688 except Exception as e: 

2689 # Do not raise, we don't want to block user action. 

2690 self._logger( 

2691 'Cannot clean volume {}: {}'.format(volume_uuid, e) 

2692 ) 

2693 

2694 # The volume can't be removed, maybe it's still in use, 

2695 # in this case rename it with the "DELETED_" prefix. 

2696 # This prefix is mandatory if it exists a snap transaction to 

2697 # rollback because the original VDI UUID can try to be renamed 

2698 # with the UUID we are trying to delete... 

2699 if not volume_uuid.startswith('DELETED_'): 

2700 self.update_volume_uuid( 

2701 volume_uuid, 'DELETED_' + volume_uuid, force=True 

2702 ) 

2703 

2704 for dest_uuid, src_uuid in updating_uuid_volumes.items(): 

2705 dest_namespace = self._build_volume_namespace(dest_uuid) 

2706 

2707 properties.namespace = dest_namespace 

2708 if int(properties.get(self.PROP_NOT_EXISTS)): 

2709 properties.clear() 

2710 continue 

2711 

2712 properties.namespace = self._build_volume_namespace(src_uuid) 

2713 properties.clear() 

2714 

2715 properties.namespace = dest_namespace 

2716 properties.pop(self.PROP_UPDATING_UUID_SRC) 

2717 

2718 if src_uuid in self._volumes: 

2719 self._volumes.remove(src_uuid) 

2720 self._volumes.add(dest_uuid) 

2721 

2722 def _get_sr_properties(self): 

2723 return self._create_linstor_kv(self._build_sr_namespace()) 

2724 

2725 def _get_volumes_by_property( 

2726 self, reg_prop, ignore_inexisting_volumes=True 

2727 ): 

2728 base_properties = self._get_kv_cache() 

2729 base_properties.namespace = self._build_volume_namespace() 

2730 

2731 volume_properties = {} 

2732 for volume_uuid in self._volumes: 

2733 volume_properties[volume_uuid] = '' 

2734 

2735 for key, value in base_properties.items(): 

2736 res = reg_prop.match(key) 

2737 if res: 

2738 volume_uuid = res.groups()[0] 

2739 if not ignore_inexisting_volumes or \ 

2740 volume_uuid in self._volumes: 

2741 volume_properties[volume_uuid] = value 

2742 

2743 return volume_properties 

2744 

2745 def _create_linstor_kv(self, namespace): 

2746 return linstor.KV( 

2747 self._group_name, 

2748 uri=self._linstor.controller_host(), 

2749 namespace=namespace 

2750 ) 

2751 

2752 def _get_volume_properties(self, volume_uuid): 

2753 properties = self._get_kv_cache() 

2754 properties.namespace = self._build_volume_namespace(volume_uuid) 

2755 return properties 

2756 

2757 @classmethod 

2758 def _list_database_backups(cls, database_backup_dir): 

2759 """ 

2760 List all visible backup files in database_backup_dir. 

2761 DATABASE_BACKUP_DIR_MAIN is only available on the Linstor Controller. 

2762 DATABASE_BACKUP_DIR_SPARE will list backups previously made when the Host was the Linstor Controller. 

2763 This may not be useful information if it is not the Controller anymore. 

2764 """ 

2765 for path in database_backup_dir.glob(DATABASE_BACKUP_NAME_FORMAT.format( 

2766 "[0-9]" * 8 + "_" + "[0-9]" * 6, "*") + ".zip"): 

2767 try: 

2768 yield path, datetime.strptime(path.name.split("-")[1], DATABASE_BACKUP_DATE_FORMAT) 

2769 except (ValueError, IndexError): 

2770 continue 

2771 

2772 @classmethod 

2773 def _get_sorted_database_backup(cls, database_backup_dir): 

2774 """ 

2775 Return list of backups in database_backup_dir, alongside their creation date. 

2776 Sorted by date from the more recent to the older one. 

2777 """ 

2778 return sorted(cls._list_database_backups(database_backup_dir), 

2779 reverse=True, 

2780 key=lambda p: p[1]) 

2781 

2782 @classmethod 

2783 def _get_latest_database_backup(cls): 

2784 """ 

2785 Return the latest backup in DATABASE_BACKUP_DIR_MAIN, and its creation date. 

2786 Returns (None, timestamp(0)) when none are found. 

2787 None will be found if it is not called on the Linstor Controller. 

2788 (cf _list_database_backups) 

2789 """ 

2790 return max(cls._list_database_backups(DATABASE_BACKUP_DIR_MAIN), 

2791 default=(None, datetime.fromtimestamp(0)), 

2792 key=lambda p: p[1]) 

2793 

2794 @classmethod 

2795 def _check_database_backup(cls, database_backup_path): 

2796 """ 

2797 Make some validation of a database backup zip-file. 

2798 Check its a valid zipfile, and CRC-test its content. 

2799 Check it contains a non-empty linstordb.mv.db file. 

2800 Always raises a LinstorDatabaseBackupError if checks failed. 

2801 """ 

2802 try: 

2803 with zipfile.ZipFile(database_backup_path, mode="r") as archive: 

2804 if archive.testzip() is not None: 

2805 raise LinstorDatabaseBackupError("zip archive CRC failed") 

2806 linstordb = next(( 

2807 f 

2808 for f in archive.filelist 

2809 if f.filename == "linstordb.mv.db" 

2810 ), None) 

2811 if not linstordb: 

2812 raise LinstorDatabaseBackupError("cannot find linstordb.mv.db") 

2813 if linstordb.file_size == 0: 

2814 raise LinstorDatabaseBackupError("linstordb.mv.db is empty") 

2815 except (FileNotFoundError, zipfile.BadZipFile, zipfile.LargeZipFile) as e: 

2816 raise LinstorDatabaseBackupError(e) from e 

2817 

2818 @classmethod 

2819 def _build_sr_namespace(cls): 

2820 return '/{}/'.format(cls.NAMESPACE_SR) 

2821 

2822 @classmethod 

2823 def _build_volume_namespace(cls, volume_uuid=None): 

2824 # Return a path to all volumes if `volume_uuid` is not given. 

2825 if volume_uuid is None: 

2826 return '/{}/'.format(cls.NAMESPACE_VOLUME) 

2827 return '/{}/{}/'.format(cls.NAMESPACE_VOLUME, volume_uuid) 

2828 

2829 @classmethod 

2830 def _get_error_str(cls, result): 

2831 return ', '.join([ 

2832 err.message for err in cls._filter_errors(result) 

2833 ]) 

2834 

2835 @classmethod 

2836 def _create_linstor_instance( 

2837 cls, uri, keep_uri_unmodified=False, attempt_count=30 

2838 ): 

2839 retry = False 

2840 

2841 def connect(uri): 

2842 if not uri: 

2843 uri = get_controller_uri() 

2844 if not uri: 

2845 raise LinstorVolumeManagerError( 

2846 'Unable to find controller uri...' 

2847 ) 

2848 instance = linstor.Linstor(uri, keep_alive=True) 

2849 instance.connect() 

2850 return instance 

2851 

2852 try: 

2853 return connect(uri) 

2854 except (linstor.errors.LinstorNetworkError, LinstorVolumeManagerError): 

2855 pass 

2856 

2857 if not keep_uri_unmodified: 

2858 uri = None 

2859 

2860 return util.retry( 

2861 lambda: connect(uri), 

2862 maxretry=attempt_count, 

2863 period=1, 

2864 exceptions=[ 

2865 linstor.errors.LinstorNetworkError, 

2866 LinstorVolumeManagerError 

2867 ] 

2868 ) 

2869 

2870 @classmethod 

2871 def _configure_volume_peer_slots(cls, lin, volume_name): 

2872 result = lin.resource_dfn_modify(volume_name, {}, peer_slots=3) 

2873 error_str = cls._get_error_str(result) 

2874 if error_str: 

2875 raise LinstorVolumeManagerError( 

2876 'Could not configure volume peer slots of {}: {}' 

2877 .format(volume_name, error_str) 

2878 ) 

2879 

2880 @classmethod 

2881 def _activate_device_path(cls, lin, node_name, volume_name): 

2882 result = lin.resource_make_available(node_name, volume_name, diskful=False) 

2883 if linstor.Linstor.all_api_responses_no_error(result): 

2884 return 

2885 errors = linstor.Linstor.filter_api_call_response_errors(result) 

2886 if len(errors) == 1 and errors[0].is_error( 

2887 linstor.consts.FAIL_EXISTS_RSC 

2888 ): 

2889 return 

2890 

2891 raise LinstorVolumeManagerError( 

2892 'Unable to activate device path of `{}` on node `{}`: {}' 

2893 .format(volume_name, node_name, ', '.join( 

2894 [str(x) for x in result])) 

2895 ) 

2896 

2897 @classmethod 

2898 def _request_database_path(cls, lin, activate=False): 

2899 node_name = socket.gethostname() 

2900 

2901 try: 

2902 resource = next(filter( 

2903 lambda resource: resource.node_name == node_name and 

2904 resource.name == DATABASE_VOLUME_NAME, 

2905 lin.resource_list_raise().resources 

2906 ), None) 

2907 except Exception as e: 

2908 raise LinstorVolumeManagerError( 

2909 'Unable to fetch database resource: {}' 

2910 .format(e) 

2911 ) 

2912 

2913 if not resource: 

2914 if activate: 

2915 cls._activate_device_path( 

2916 lin, node_name, DATABASE_VOLUME_NAME 

2917 ) 

2918 return cls._request_database_path( 

2919 DATABASE_VOLUME_NAME, DATABASE_VOLUME_NAME 

2920 ) 

2921 raise LinstorVolumeManagerError( 

2922 'Empty dev path for `{}`, but definition "seems" to exist' 

2923 .format(DATABASE_PATH) 

2924 ) 

2925 # Contains a path of the /dev/drbd<id> form. 

2926 return resource.volumes[0].device_path 

2927 

2928 @classmethod 

2929 def _create_database_volume( 

2930 cls, lin, group_name, storage_pool_name, node_names, redundancy 

2931 ): 

2932 try: 

2933 dfns = lin.resource_dfn_list_raise().resource_definitions 

2934 except Exception as e: 

2935 raise LinstorVolumeManagerError( 

2936 'Unable to get definitions during database creation: {}' 

2937 .format(e) 

2938 ) 

2939 

2940 if dfns: 

2941 raise LinstorVolumeManagerError( 

2942 'Could not create volume `{}` from SR `{}`, '.format( 

2943 DATABASE_VOLUME_NAME, group_name 

2944 ) + 'LINSTOR volume list must be empty.' 

2945 ) 

2946 

2947 # Workaround to use thin lvm. Without this line an error is returned: 

2948 # "Not enough available nodes" 

2949 # I don't understand why but this command protect against this bug. 

2950 try: 

2951 pools = lin.storage_pool_list_raise( 

2952 filter_by_stor_pools=[storage_pool_name] 

2953 ) 

2954 except Exception as e: 

2955 raise LinstorVolumeManagerError( 

2956 'Failed to get storage pool list before database creation: {}' 

2957 .format(e) 

2958 ) 

2959 

2960 # Ensure we have a correct list of storage pools. 

2961 assert pools.storage_pools # We must have at least one storage pool! 

2962 nodes_with_pool = list(map(lambda pool: pool.node_name, pools.storage_pools)) 

2963 for node_name in nodes_with_pool: 

2964 assert node_name in node_names 

2965 util.SMlog('Nodes with storage pool: {}'.format(nodes_with_pool)) 

2966 

2967 # Create the database definition. 

2968 size = cls.round_up_volume_size(DATABASE_SIZE) 

2969 cls._check_volume_creation_errors(lin.resource_group_spawn( 

2970 rsc_grp_name=group_name, 

2971 rsc_dfn_name=DATABASE_VOLUME_NAME, 

2972 vlm_sizes=['{}B'.format(size)], 

2973 definitions_only=True 

2974 ), DATABASE_VOLUME_NAME, group_name) 

2975 cls._configure_volume_peer_slots(lin, DATABASE_VOLUME_NAME) 

2976 

2977 # Create real resources on the first nodes. 

2978 resources = [] 

2979 

2980 diskful_nodes = [] 

2981 diskless_nodes = [] 

2982 for node_name in node_names: 

2983 if node_name in nodes_with_pool: 

2984 diskful_nodes.append(node_name) 

2985 else: 

2986 diskless_nodes.append(node_name) 

2987 

2988 assert diskful_nodes 

2989 for node_name in diskful_nodes[:redundancy]: 

2990 util.SMlog('Create database diskful on {}'.format(node_name)) 

2991 resources.append(linstor.ResourceData( 

2992 node_name=node_name, 

2993 rsc_name=DATABASE_VOLUME_NAME, 

2994 storage_pool=storage_pool_name 

2995 )) 

2996 # Create diskless resources on the remaining set. 

2997 for node_name in diskful_nodes[redundancy:] + diskless_nodes: 

2998 util.SMlog('Create database diskless on {}'.format(node_name)) 

2999 resources.append(linstor.ResourceData( 

3000 node_name=node_name, 

3001 rsc_name=DATABASE_VOLUME_NAME, 

3002 diskless=True 

3003 )) 

3004 

3005 result = lin.resource_create(resources) 

3006 error_str = cls._get_error_str(result) 

3007 if error_str: 

3008 raise LinstorVolumeManagerError( 

3009 'Could not create database volume from SR `{}`: {}'.format( 

3010 group_name, error_str 

3011 ) 

3012 ) 

3013 

3014 # Create database and ensure path exists locally and 

3015 # on replicated devices. 

3016 current_device_path = cls._request_database_path(lin, activate=True) 

3017 

3018 # Ensure diskless paths exist on other hosts. Otherwise PBDs can't be 

3019 # plugged. 

3020 for node_name in node_names: 

3021 cls._activate_device_path(lin, node_name, DATABASE_VOLUME_NAME) 

3022 

3023 # We use realpath here to get the /dev/drbd<id> path instead of 

3024 # /dev/drbd/by-res/<resource_name>. 

3025 expected_device_path = cls.build_device_path(DATABASE_VOLUME_NAME) 

3026 util.wait_for_path(expected_device_path, 5) 

3027 

3028 device_realpath = os.path.realpath(expected_device_path) 

3029 if current_device_path != device_realpath: 

3030 raise LinstorVolumeManagerError( 

3031 'Invalid path, current={}, expected={} (realpath={})' 

3032 .format( 

3033 current_device_path, 

3034 expected_device_path, 

3035 device_realpath 

3036 ) 

3037 ) 

3038 

3039 try: 

3040 util.retry( 

3041 lambda: util.pread2([DATABASE_MKFS, expected_device_path]), 

3042 maxretry=5 

3043 ) 

3044 except Exception as e: 

3045 raise LinstorVolumeManagerError( 

3046 'Failed to execute {} on database volume: {}' 

3047 .format(DATABASE_MKFS, e) 

3048 ) 

3049 

3050 return expected_device_path 

3051 

3052 @classmethod 

3053 def _destroy_database_volume(cls, lin, group_name): 

3054 error_str = cls._get_error_str( 

3055 lin.resource_dfn_delete(DATABASE_VOLUME_NAME) 

3056 ) 

3057 if error_str: 

3058 raise LinstorVolumeManagerError( 

3059 'Could not destroy resource `{}` from SR `{}`: {}' 

3060 .format(DATABASE_VOLUME_NAME, group_name, error_str) 

3061 ) 

3062 

3063 @classmethod 

3064 def _mount_database_volume(cls, volume_path, mount=True, force=False): 

3065 try: 

3066 # 1. Create a backup config folder. 

3067 database_not_empty = bool(os.listdir(DATABASE_PATH)) 

3068 backup_path = cls._create_database_backup_path() 

3069 

3070 # 2. Move the config in the mounted volume. 

3071 if database_not_empty: 

3072 cls._move_files(DATABASE_PATH, backup_path) 

3073 

3074 cls._mount_volume(volume_path, DATABASE_PATH, mount) 

3075 

3076 if database_not_empty: 

3077 cls._move_files(backup_path, DATABASE_PATH, force) 

3078 

3079 # 3. Remove useless backup directory. 

3080 try: 

3081 os.rmdir(backup_path) 

3082 except Exception as e: 

3083 raise LinstorVolumeManagerError( 

3084 'Failed to remove backup path {} of LINSTOR config: {}' 

3085 .format(backup_path, e) 

3086 ) 

3087 except Exception as e: 

3088 def force_exec(fn): 

3089 try: 

3090 fn() 

3091 except Exception: 

3092 pass 

3093 

3094 if mount == cls.is_controller(): 

3095 force_exec(lambda: cls._move_files( 

3096 DATABASE_PATH, backup_path 

3097 )) 

3098 force_exec(lambda: cls._mount_volume( 

3099 volume_path, DATABASE_PATH, not mount 

3100 )) 

3101 

3102 if mount != cls.is_controller(): 

3103 force_exec(lambda: cls._move_files( 

3104 backup_path, DATABASE_PATH 

3105 )) 

3106 

3107 force_exec(lambda: os.rmdir(backup_path)) 

3108 raise e 

3109 

3110 @classmethod 

3111 def _force_destroy_database_volume(cls, lin, group_name): 

3112 try: 

3113 cls._destroy_database_volume(lin, group_name) 

3114 except Exception: 

3115 pass 

3116 

3117 @classmethod 

3118 def _destroy_storage_pool(cls, lin, group_name, node_name): 

3119 def destroy(): 

3120 result = lin.storage_pool_delete(node_name, group_name) 

3121 errors = cls._filter_errors(result) 

3122 if cls._check_errors(errors, [ 

3123 linstor.consts.FAIL_NOT_FOUND_STOR_POOL, 

3124 linstor.consts.FAIL_NOT_FOUND_STOR_POOL_DFN 

3125 ]): 

3126 return 

3127 

3128 if errors: 

3129 raise LinstorVolumeManagerError( 

3130 'Failed to destroy SP `{}` on node `{}`: {}'.format( 

3131 group_name, 

3132 node_name, 

3133 cls._get_error_str(errors) 

3134 ) 

3135 ) 

3136 

3137 # We must retry to avoid errors like: 

3138 # "can not be deleted as volumes / snapshot-volumes are still using it" 

3139 # after LINSTOR database volume destruction. 

3140 return util.retry(destroy, maxretry=10) 

3141 

3142 @classmethod 

3143 def _create_resource_group( 

3144 cls, 

3145 lin, 

3146 group_name, 

3147 storage_pool_name, 

3148 redundancy, 

3149 destroy_old_group 

3150 ): 

3151 rg_creation_attempt = 0 

3152 while True: 

3153 result = lin.resource_group_create( 

3154 name=group_name, 

3155 place_count=redundancy, 

3156 storage_pool=storage_pool_name, 

3157 diskless_on_remaining=False 

3158 ) 

3159 error_str = cls._get_error_str(result) 

3160 if not error_str: 

3161 break 

3162 

3163 errors = cls._filter_errors(result) 

3164 if destroy_old_group and cls._check_errors(errors, [ 

3165 linstor.consts.FAIL_EXISTS_RSC_GRP 

3166 ]): 

3167 rg_creation_attempt += 1 

3168 if rg_creation_attempt < 2: 

3169 try: 

3170 cls._destroy_resource_group(lin, group_name) 

3171 except Exception as e: 

3172 error_str = 'Failed to destroy old and empty RG: {}'.format(e) 

3173 else: 

3174 continue 

3175 

3176 raise LinstorVolumeManagerError( 

3177 'Could not create RG `{}`: {}'.format( 

3178 group_name, error_str 

3179 ) 

3180 ) 

3181 

3182 result = lin.volume_group_create(group_name) 

3183 error_str = cls._get_error_str(result) 

3184 if error_str: 

3185 raise LinstorVolumeManagerError( 

3186 'Could not create VG `{}`: {}'.format( 

3187 group_name, error_str 

3188 ) 

3189 ) 

3190 

3191 @classmethod 

3192 def _destroy_resource_group(cls, lin, group_name): 

3193 def destroy(): 

3194 result = lin.resource_group_delete(group_name) 

3195 errors = cls._filter_errors(result) 

3196 if cls._check_errors(errors, [ 

3197 linstor.consts.FAIL_NOT_FOUND_RSC_GRP 

3198 ]): 

3199 return 

3200 

3201 if errors: 

3202 raise LinstorVolumeManagerError( 

3203 'Failed to destroy RG `{}`: {}' 

3204 .format(group_name, cls._get_error_str(errors)) 

3205 ) 

3206 

3207 return util.retry(destroy, maxretry=10) 

3208 

3209 @classmethod 

3210 def build_group_name(cls, base_name): 

3211 # If thin provisioning is used we have a path like this: 

3212 # `VG/LV`. "/" is not accepted by LINSTOR. 

3213 return '{}{}'.format(cls.PREFIX_SR, base_name.replace('/', '_')) 

3214 

3215 # Used to store important data in a HA context, 

3216 # i.e. a replication count of 3. 

3217 @classmethod 

3218 def _build_ha_group_name(cls, base_name): 

3219 return '{}{}'.format(cls.PREFIX_HA, base_name.replace('/', '_')) 

3220 

3221 @classmethod 

3222 def _check_volume_creation_errors(cls, result, volume_uuid, group_name): 

3223 errors = cls._filter_errors(result) 

3224 if cls._check_errors(errors, [ 

3225 linstor.consts.FAIL_EXISTS_RSC, linstor.consts.FAIL_EXISTS_RSC_DFN 

3226 ]): 

3227 raise LinstorVolumeManagerError( 

3228 'Failed to create volume `{}` from SR `{}`, it already exists' 

3229 .format(volume_uuid, group_name), 

3230 LinstorVolumeManagerError.ERR_VOLUME_EXISTS 

3231 ) 

3232 

3233 if cls._check_errors(errors, [linstor.consts.FAIL_NOT_FOUND_RSC_GRP]): 

3234 raise LinstorVolumeManagerError( 

3235 'Failed to create volume `{}` from SR `{}`, resource group doesn\'t exist' 

3236 .format(volume_uuid, group_name), 

3237 LinstorVolumeManagerError.ERR_GROUP_NOT_EXISTS 

3238 ) 

3239 

3240 if errors: 

3241 raise LinstorVolumeManagerError( 

3242 'Failed to create volume `{}` from SR `{}`: {}'.format( 

3243 volume_uuid, 

3244 group_name, 

3245 cls._get_error_str(errors) 

3246 ) 

3247 ) 

3248 

3249 @classmethod 

3250 def _move_files(cls, src_dir, dest_dir, force=False): 

3251 def listdir(dir): 

3252 ignored = ['lost+found'] 

3253 return [file for file in os.listdir(dir) if file not in ignored] 

3254 

3255 try: 

3256 if not force: 

3257 files = listdir(dest_dir) 

3258 if files: 

3259 raise LinstorVolumeManagerError( 

3260 'Cannot move files from {} to {} because destination ' 

3261 'contains: {}'.format(src_dir, dest_dir, files) 

3262 ) 

3263 except LinstorVolumeManagerError: 

3264 raise 

3265 except Exception as e: 

3266 raise LinstorVolumeManagerError( 

3267 'Cannot list dir {}: {}'.format(dest_dir, e) 

3268 ) 

3269 

3270 try: 

3271 for file in listdir(src_dir): 

3272 try: 

3273 dest_file = os.path.join(dest_dir, file) 

3274 if not force and os.path.exists(dest_file): 

3275 raise LinstorVolumeManagerError( 

3276 'Cannot move {} because it already exists in the ' 

3277 'destination'.format(file) 

3278 ) 

3279 shutil.move(os.path.join(src_dir, file), dest_file) 

3280 except LinstorVolumeManagerError: 

3281 raise 

3282 except Exception as e: 

3283 raise LinstorVolumeManagerError( 

3284 'Cannot move {}: {}'.format(file, e) 

3285 ) 

3286 except Exception as e: 

3287 if not force: 

3288 try: 

3289 cls._move_files(dest_dir, src_dir, force=True) 

3290 except Exception: 

3291 pass 

3292 

3293 raise LinstorVolumeManagerError( 

3294 'Failed to move files from {} to {}: {}'.format( 

3295 src_dir, dest_dir, e 

3296 ) 

3297 ) 

3298 

3299 @staticmethod 

3300 def _create_database_backup_path(): 

3301 path = DATABASE_PATH + '-' + str(uuid.uuid4()) 

3302 try: 

3303 os.mkdir(path) 

3304 return path 

3305 except Exception as e: 

3306 raise LinstorVolumeManagerError( 

3307 'Failed to create backup path {} of LINSTOR config: {}' 

3308 .format(path, e) 

3309 ) 

3310 

3311 @staticmethod 

3312 def _get_filtered_properties(properties): 

3313 return dict(properties.items()) 

3314 

3315 @staticmethod 

3316 def _filter_errors(result): 

3317 return [ 

3318 err for err in result 

3319 if hasattr(err, 'is_error') and err.is_error() 

3320 ] 

3321 

3322 @staticmethod 

3323 def _check_errors(result, codes): 

3324 for err in result: 

3325 for code in codes: 

3326 if err.is_error(code): 

3327 return True 

3328 return False 

3329 

3330 @classmethod 

3331 def _controller_is_running(cls): 

3332 return cls._service_is_running('linstor-controller') 

3333 

3334 @classmethod 

3335 def _start_controller(cls, start=True): 

3336 return cls._start_service('linstor-controller', start) 

3337 

3338 @staticmethod 

3339 def _start_service(name, start=True): 

3340 action = 'start' if start else 'stop' 

3341 (ret, out, err) = util.doexec([ 

3342 'systemctl', action, name 

3343 ]) 

3344 if ret != 0: 

3345 raise LinstorVolumeManagerError( 

3346 'Failed to {} {}: {} {}' 

3347 .format(action, name, out, err) 

3348 ) 

3349 

3350 @staticmethod 

3351 def _service_is_running(name): 

3352 (ret, out, err) = util.doexec([ 

3353 'systemctl', 'is-active', '--quiet', name 

3354 ]) 

3355 return not ret 

3356 

3357 @staticmethod 

3358 def _is_mounted(mountpoint): 

3359 (ret, out, err) = util.doexec(['mountpoint', '-q', mountpoint]) 

3360 return ret == 0 

3361 

3362 @classmethod 

3363 def _mount_volume(cls, volume_path, mountpoint, mount=True): 

3364 if mount: 

3365 try: 

3366 util.pread(['mount', volume_path, mountpoint]) 

3367 except Exception as e: 

3368 raise LinstorVolumeManagerError( 

3369 'Failed to mount volume {} on {}: {}' 

3370 .format(volume_path, mountpoint, e) 

3371 ) 

3372 else: 

3373 try: 

3374 if cls._is_mounted(mountpoint): 

3375 util.pread(['umount', mountpoint]) 

3376 except Exception as e: 

3377 raise LinstorVolumeManagerError( 

3378 'Failed to umount volume {} on {}: {}' 

3379 .format(volume_path, mountpoint, e) 

3380 ) 

3381 

3382 

3383# ============================================================================== 

3384 

3385# Check if a path is a DRBD resource and log the process name/pid 

3386# that opened it. 

3387def log_drbd_openers(path): 

3388 # Ignore if it's not a symlink to DRBD resource. 

3389 if not path.startswith(DRBD_BY_RES_PATH): 

3390 return 

3391 

3392 # Compute resource name. 

3393 res_name_end = path.find('/', len(DRBD_BY_RES_PATH)) 

3394 if res_name_end == -1: 

3395 return 

3396 res_name = path[len(DRBD_BY_RES_PATH):res_name_end] 

3397 

3398 volume_end = path.rfind('/') 

3399 if volume_end == res_name_end: 

3400 return 

3401 volume = path[volume_end + 1:] 

3402 

3403 try: 

3404 # Ensure path is a DRBD. 

3405 drbd_path = os.path.realpath(path) 

3406 stats = os.stat(drbd_path) 

3407 if not stat.S_ISBLK(stats.st_mode) or os.major(stats.st_rdev) != 147: 

3408 return 

3409 

3410 # Find where the device is open. 

3411 (ret, stdout, stderr) = util.doexec(['drbdadm', 'status', res_name]) 

3412 if ret != 0: 

3413 util.SMlog('Failed to execute `drbdadm status` on `{}`: {}'.format( 

3414 res_name, stderr 

3415 )) 

3416 return 

3417 

3418 # Is it a local device? 

3419 if stdout.startswith('{} role:Primary'.format(res_name)): 

3420 util.SMlog( 

3421 'DRBD resource `{}` is open on local host: {}' 

3422 .format(path, get_local_volume_openers(res_name, volume)) 

3423 ) 

3424 return 

3425 

3426 # Is it a remote device? 

3427 util.SMlog( 

3428 'DRBD resource `{}` is open on hosts: {}' 

3429 .format(path, get_all_volume_openers(res_name, volume)) 

3430 ) 

3431 except Exception as e: 

3432 util.SMlog( 

3433 'Got exception while trying to determine where DRBD resource ' + 

3434 '`{}` is open: {}'.format(path, e) 

3435 )