Coverage for drivers/linstorvolumemanager.py : 11%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python3
2#
3# Copyright (C) 2020 Vates SAS - ronan.abhamon@vates.fr
4#
5# This program is free software: you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation, either version 3 of the License, or
8# (at your option) any later version.
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program. If not, see <https://www.gnu.org/licenses/>.
16#
18from sm_typing import (
19 Any,
20 Dict,
21 List,
22 cast,
23 override,
24)
26import json
27import linstor
28import os.path
29import re
30import shutil
31import socket
32import stat
33import time
34import util
35import uuid
36from datetime import datetime
37from pathlib import Path
38import contextlib
39import zipfile
41# Persistent prefix to add to RAW persistent volumes.
42PERSISTENT_PREFIX = 'xcp-persistent-'
44# Contains the data of the "/var/lib/linstor" directory.
45DATABASE_VOLUME_NAME = PERSISTENT_PREFIX + 'database'
46DATABASE_SIZE = 1 << 30 # 1GB.
47DATABASE_PATH = '/var/lib/linstor'
48DATABASE_MKFS = 'mkfs.ext4'
49DATABASE_BACKUP_DIR_MAIN = Path(DATABASE_PATH)
50DATABASE_BACKUP_DIR_SPARE = Path("/var/lib/linstor.d/db-backups")
51DATABASE_BACKUP_NAME_FORMAT = "linstor_database_backup-{}-{}"
52DATABASE_BACKUP_RETENTION = 10
53DATABASE_BACKUP_DATE_FORMAT = "%Y%m%d_%H%M%S"
54LINSTOR_SATELLITE_PORT = 3366
56REG_DRBDADM_PRIMARY = re.compile("([^\\s]+)\\s+role:Primary")
57REG_DRBDSETUP_IP = re.compile('[^\\s]+\\s+(.*):.*$')
59DRBD_BY_RES_PATH = '/dev/drbd/by-res/'
60DRBD_OPENERS_PATH_FORMAT = '/sys/kernel/debug/drbd/resources/{}/volumes/{}/openers'
62PLUGIN = 'linstor-manager'
64LinstorLocalVolumeOpeners = Dict[str, Dict[str, Any]]
65LinstorVolumeOpeners = Dict[str, LinstorLocalVolumeOpeners]
67# ==============================================================================
69def has_local_volume_openers(resource_name, volume) -> bool:
70 if not resource_name or volume is None:
71 raise Exception('Cannot check for DRBD openers without resource name and/or volume.')
73 path = DRBD_OPENERS_PATH_FORMAT.format(resource_name, volume)
74 openers_char = ''
76 try:
77 with open(path, 'r') as openers:
78 # Read a single character to check if there is any opener listed.
79 # If we get nothing back, then the file is empty meaning there are
80 # no openers.
81 openers_char = openers.read(1)
82 except FileNotFoundError:
83 return False
85 return openers_char != ''
87def get_local_volume_openers(resource_name, volume) -> LinstorLocalVolumeOpeners:
88 if not resource_name or volume is None:
89 raise Exception('Cannot get DRBD openers without resource name and/or volume.')
91 path = DRBD_OPENERS_PATH_FORMAT.format(resource_name, volume)
93 with open(path, 'r') as openers:
94 # Not a big cost, so read all lines directly.
95 lines = openers.readlines()
97 result = {}
99 opener_re = re.compile('(.*)\\s+([0-9]+)\\s+([0-9]+)')
100 for line in lines:
101 match = opener_re.match(line)
102 assert match
104 groups = match.groups()
105 process_name = groups[0]
106 pid = groups[1]
107 open_duration_ms = groups[2]
109 try:
110 cmdline = util.get_process_cmdline(int(pid))
111 except Exception as e:
112 util.SMlog(f"Failed to get command line of `{pid}`: {e}")
113 cmdline = []
115 result[pid] = {
116 'process-name': process_name,
117 'open-duration': open_duration_ms,
118 'cmdline': cmdline
119 }
121 return cast(LinstorLocalVolumeOpeners, json.dumps(result))
123def get_all_volume_openers(resource_name, volume) -> LinstorVolumeOpeners:
124 PLUGIN_CMD = 'getDrbdOpeners'
126 volume = str(volume)
127 openers = {}
129 with util.ApiSession("SM-get-linstor-volume-openers") as session:
130 hosts = session.xenapi.host.get_all_records()
131 for host_ref, host_record in hosts.items():
132 node_name = host_record['hostname']
133 try:
134 if not session.xenapi.host_metrics.get_record(
135 host_record['metrics']
136 )['live']:
137 # Ensure we call plugin on online hosts only.
138 continue
140 openers[node_name] = json.loads(
141 session.xenapi.host.call_plugin(host_ref, PLUGIN, PLUGIN_CMD, {
142 'resourceName': resource_name,
143 'volume': volume
144 })
145 )
146 except Exception as e:
147 util.SMlog('Failed to get openers of `{}` on `{}`: {}'.format(
148 resource_name, node_name, e
149 ))
151 return openers
154# ==============================================================================
156def round_up(value, divisor):
157 assert divisor
158 divisor = int(divisor)
159 return ((int(value) + divisor - 1) // divisor) * divisor
162def round_down(value, divisor):
163 assert divisor
164 value = int(value)
165 return value - (value % int(divisor))
168# ==============================================================================
170def _get_controller_addresses() -> List[str]:
171 try:
172 (ret, stdout, stderr) = util.doexec([
173 "/usr/sbin/ss", "-tnH", "state", "established", f"( sport = :{LINSTOR_SATELLITE_PORT} )"
174 ])
175 if ret == 0:
176 return [
177 line.split()[3].rsplit(":", 1)[0]
178 for line in stdout.splitlines()
179 ]
180 util.SMlog(f"Unexpected code {ret}: {stderr}")
181 except Exception as e:
182 util.SMlog(f"Unable to get controller addresses: {e}")
183 return []
185def _get_controller_uri() -> str:
186 # TODO: Check that an IP address from the current pool is returned.
187 addresses = _get_controller_addresses()
188 return "linstor://" + addresses[0] if addresses else ""
190def get_controller_uri():
191 retries = 0
192 while True:
193 uri = _get_controller_uri()
194 if uri:
195 return uri
197 retries += 1
198 if retries >= 30:
199 break
200 time.sleep(1)
203def get_controller_node_name():
204 PLUGIN_CMD = 'hasControllerRunning'
206 (ret, stdout, stderr) = util.doexec([
207 'drbdadm', 'status', DATABASE_VOLUME_NAME
208 ])
210 if ret == 0:
211 if stdout.startswith('{} role:Primary'.format(DATABASE_VOLUME_NAME)):
212 return 'localhost'
214 res = REG_DRBDADM_PRIMARY.search(stdout)
215 if res:
216 return res.groups()[0]
218 session = util.timeout(5, util.ApiSession, "SM-get-linstor-controller-node-name").session
220 for host_ref, host_record in session.xenapi.host.get_all_records().items():
221 node_name = host_record['hostname']
222 try:
223 if not session.xenapi.host_metrics.get_record(
224 host_record['metrics']
225 )['live']:
226 continue
228 if util.strtobool(session.xenapi.host.call_plugin(
229 host_ref, PLUGIN, PLUGIN_CMD, {}
230 )):
231 return node_name
232 except Exception as e:
233 util.SMlog('Failed to call plugin to get controller on `{}`: {}'.format(
234 node_name, e
235 ))
238def demote_drbd_resource(node_name, resource_name):
239 PLUGIN_CMD = 'demoteDrbdResource'
241 session = util.timeout(5, util.ApiSession, "SM-demote-drbd-resource").session
243 for host_ref, host_record in session.xenapi.host.get_all_records().items():
244 if host_record['hostname'] != node_name:
245 continue
247 try:
248 session.xenapi.host.call_plugin(
249 host_ref, PLUGIN, PLUGIN_CMD, {'resource_name': resource_name}
250 )
251 except Exception as e:
252 util.SMlog('Failed to demote resource `{}` on `{}`: {}'.format(
253 resource_name, node_name, e
254 ))
255 raise Exception(
256 'Can\'t demote resource `{}`, unable to find node `{}`'
257 .format(resource_name, node_name)
258 )
260# ==============================================================================
262class LinstorVolumeManagerError(Exception):
263 ERR_GENERIC = 0,
264 ERR_VOLUME_EXISTS = 1,
265 ERR_VOLUME_NOT_EXISTS = 2,
266 ERR_VOLUME_DESTROY = 3,
267 ERR_GROUP_NOT_EXISTS = 4,
268 ERR_VOLUME_IN_USE = 5,
269 ERR_REPAIR_REQUIRED = 6,
271 def __init__(self, message, code=ERR_GENERIC):
272 super(LinstorVolumeManagerError, self).__init__(message)
273 self._code = code
275 @property
276 def code(self):
277 return self._code
279class LinstorDatabaseBackupError(Exception):
280 pass
282# ==============================================================================
284# Note:
285# If a storage pool is not accessible after a network change:
286# linstor node interface modify <NODE> default --ip <IP>
289class LinstorVolumeManager(object):
290 """
291 API to manager LINSTOR volumes in XCP-ng.
292 A volume in this context is a physical part of the storage layer.
293 """
295 __slots__ = (
296 '_linstor', '_uri', '_logger', '_redundancy',
297 '_base_group_name', '_group_name', '_ha_group_name',
298 '_volumes', '_storage_pools', '_storage_pools_time',
299 '_kv_cache', '_resource_cache', '_volume_info_cache',
300 '_kv_cache_dirty', '_resource_cache_dirty', '_volume_info_cache_dirty',
301 '_resources_info_cache',
302 )
304 DEV_ROOT_PATH = DRBD_BY_RES_PATH
306 # Default sector size.
307 BLOCK_SIZE = 512
309 # List of volume properties.
310 PROP_METADATA = 'metadata'
311 PROP_NOT_EXISTS = 'not-exists'
312 PROP_VOLUME_NAME = 'volume-name'
313 PROP_IS_READONLY_TIMESTAMP = 'readonly-timestamp'
315 # A volume can only be locked for a limited duration.
316 # The goal is to give enough time to slaves to execute some actions on
317 # a device before an UUID update or a coalesce for example.
318 # Expiration is expressed in seconds.
319 LOCKED_EXPIRATION_DELAY = 1 * 60
321 # Used when volume uuid is being updated.
322 PROP_UPDATING_UUID_SRC = 'updating-uuid-src'
324 # States of property PROP_NOT_EXISTS.
325 STATE_EXISTS = '0'
326 STATE_NOT_EXISTS = '1'
327 STATE_CREATING = '2'
329 # Property namespaces.
330 NAMESPACE_SR = 'xcp/sr'
331 NAMESPACE_VOLUME = 'xcp/volume'
333 # Regex to match properties.
334 REG_PROP = '^([^/]+)/{}$'
336 REG_METADATA = re.compile(REG_PROP.format(PROP_METADATA))
337 REG_NOT_EXISTS = re.compile(REG_PROP.format(PROP_NOT_EXISTS))
338 REG_VOLUME_NAME = re.compile(REG_PROP.format(PROP_VOLUME_NAME))
339 REG_UPDATING_UUID_SRC = re.compile(REG_PROP.format(PROP_UPDATING_UUID_SRC))
341 # Prefixes of SR/VOLUME in the LINSTOR DB.
342 # A LINSTOR (resource, group, ...) name cannot start with a number.
343 # So we add a prefix behind our SR/VOLUME uuids.
344 PREFIX_SR = 'xcp-sr-'
345 PREFIX_HA = 'xcp-ha-'
346 PREFIX_VOLUME = 'xcp-volume-'
348 # Limit request number when storage pool info is asked, we fetch
349 # the current pool status after N elapsed seconds.
350 STORAGE_POOLS_FETCH_INTERVAL = 15
352 @staticmethod
353 def default_logger(*args):
354 print(args)
356 # --------------------------------------------------------------------------
357 # API.
358 # --------------------------------------------------------------------------
360 class VolumeInfo(object):
361 __slots__ = (
362 'name',
363 'allocated_size', # Allocated size, place count is not used.
364 'virtual_size', # Total virtual available size of this volume
365 # (i.e. the user size at creation).
366 'diskful' # Array of nodes that have a diskful volume.
367 )
369 def __init__(self, name):
370 self.name = name
371 self.allocated_size = 0
372 self.virtual_size = 0
373 self.diskful = []
375 @override
376 def __repr__(self) -> str:
377 return 'VolumeInfo("{}", {}, {}, {})'.format(
378 self.name, self.allocated_size, self.virtual_size,
379 self.diskful
380 )
382 # --------------------------------------------------------------------------
384 def __init__(
385 self, uri, group_name, repair=False, logger=default_logger.__func__,
386 attempt_count=30
387 ):
388 """
389 Create a new LinstorVolumeManager object.
390 :param str uri: URI to communicate with the LINSTOR controller.
391 :param str group_name: The SR group name to use.
392 :param bool repair: If true we try to remove bad volumes due to a crash
393 or unexpected behavior.
394 :param function logger: Function to log messages.
395 :param int attempt_count: Number of attempts to join the controller.
396 """
398 self._uri = uri
399 self._linstor = self._create_linstor_instance(
400 uri, attempt_count=attempt_count
401 )
404 mismatched_nodes = [
405 node for node in self._linstor.node_list().pop().nodes if node.connection_status == "VERSION_MISMATCH"
406 ]
408 if mismatched_nodes:
409 raise LinstorVolumeManagerError(
410 "Some linstor nodes are not using the same version. " +
411 f"Incriminated nodes are: {','.join([node.name for node in mismatched_nodes])}"
412 )
414 self._base_group_name = group_name
416 # Ensure group exists.
417 group_name = self.build_group_name(group_name)
418 groups = self._linstor.resource_group_list_raise([group_name]).resource_groups
419 if not groups:
420 raise LinstorVolumeManagerError(
421 'Unable to find `{}` Linstor SR'.format(group_name)
422 )
424 # Ok. ;)
425 self._logger = logger
426 self._redundancy = groups[0].select_filter.place_count
427 self._group_name = group_name
428 self._ha_group_name = self._build_ha_group_name(self._base_group_name)
429 self._volumes = set()
430 self._storage_pools_time = 0
432 # To increase performance and limit request count to LINSTOR services,
433 # we use caches.
434 self._resource_cache = None
435 self._resource_cache_dirty = True
436 self._volume_info_cache = None
437 self._volume_info_cache_dirty = True
438 self._resources_info_cache = None
440 # While building the volume list, an error might be thrown if repairing
441 # the SR is required (e.g. a volume is in in the middle of having its
442 # UUID being updated).
443 # This may only be a transient state, so we can retry multiple times to
444 # see if we reach a healthy state.
445 retry_count = 0
446 while True:
447 try:
448 self._kv_cache = self._create_kv_cache()
449 self._build_volumes(repair=repair)
451 break
452 except LinstorVolumeManagerError as e:
453 if e.code != LinstorVolumeManagerError.ERR_REPAIR_REQUIRED or retry_count >= 10:
454 raise
456 retry_count += 1
457 self._logger(
458 f'Cannot build volume list ({e}). Retry number: {retry_count}'
459 )
461 time.sleep(0.1)
463 @property
464 def uri(self) -> str:
465 return self._uri
467 @property
468 def native_client(self) -> "linstor.Linstor":
469 return self._linstor
471 @property
472 def group_name(self):
473 """
474 Give the used group name.
475 :return: The group name.
476 :rtype: str
477 """
478 return self._base_group_name
480 @property
481 def redundancy(self):
482 """
483 Give the used redundancy.
484 :return: The redundancy.
485 :rtype: int
486 """
487 return self._redundancy
489 @property
490 def volumes(self):
491 """
492 Give the volumes uuid set.
493 :return: The volumes uuid set.
494 :rtype: set(str)
495 """
496 return self._volumes
498 @property
499 def max_volume_size_allowed(self):
500 """
501 Give the max volume size currently available in B.
502 :return: The current size.
503 :rtype: int
504 """
506 candidates = self._find_best_size_candidates()
507 if not candidates:
508 raise LinstorVolumeManagerError(
509 'Failed to get max volume size allowed'
510 )
512 size = candidates[0].max_volume_size
513 if size < 0:
514 raise LinstorVolumeManagerError(
515 'Invalid max volume size allowed given: {}'.format(size)
516 )
517 return self.round_down_volume_size(size * 1024)
519 @property
520 def physical_size(self):
521 """
522 Give the total physical size of the SR.
523 :return: The physical size.
524 :rtype: int
525 """
526 return self._compute_size('total_capacity')
528 @property
529 def physical_free_size(self):
530 """
531 Give the total free physical size of the SR.
532 :return: The physical free size.
533 :rtype: int
534 """
535 return self._compute_size('free_capacity')
537 @property
538 def allocated_volume_size(self):
539 """
540 Give the allocated size for all volumes. The place count is not
541 used here. When thick lvm is used, the size for one volume should
542 be equal to the virtual volume size. With thin lvm, the size is equal
543 or lower to the volume size.
544 :return: The allocated size of all volumes.
545 :rtype: int
546 """
548 # Paths: /res_name/vol_number/size
549 sizes = {}
551 for resource in self._get_resource_cache().resources:
552 if resource.name not in sizes:
553 current = sizes[resource.name] = {}
554 else:
555 current = sizes[resource.name]
557 for volume in resource.volumes:
558 # We ignore diskless pools of the form "DfltDisklessStorPool".
559 if volume.storage_pool_name != self._group_name:
560 continue
562 allocated_size = max(volume.allocated_size, 0)
563 current_allocated_size = current.get(volume.number) or -1
564 if allocated_size > current_allocated_size:
565 current[volume.number] = allocated_size
567 total_size = 0
568 for volumes in sizes.values():
569 for size in volumes.values():
570 total_size += size
572 return total_size * 1024
574 def get_min_physical_size(self):
575 """
576 Give the minimum physical size of the SR.
577 I.e. the size of the smallest disk + the number of pools.
578 :return: The physical min size.
579 :rtype: tuple(int, int)
580 """
581 size = None
582 pool_count = 0
583 for pool in self._get_storage_pools(force=True):
584 space = pool.free_space
585 if space:
586 pool_count += 1
587 current_size = space.total_capacity
588 if current_size < 0:
589 raise LinstorVolumeManagerError(
590 'Failed to get pool total_capacity attr of `{}`'
591 .format(pool.node_name)
592 )
593 if size is None or current_size < size:
594 size = current_size
595 return (pool_count, (size or 0) * 1024)
597 @property
598 def metadata(self):
599 """
600 Get the metadata of the SR.
601 :return: Dictionary that contains metadata.
602 :rtype: dict(str, dict)
603 """
605 sr_properties = self._get_sr_properties()
606 metadata = sr_properties.get(self.PROP_METADATA)
607 if metadata is not None:
608 metadata = json.loads(metadata)
609 if isinstance(metadata, dict):
610 return metadata
611 raise LinstorVolumeManagerError(
612 'Expected dictionary in SR metadata: {}'.format(
613 self._group_name
614 )
615 )
617 return {}
619 @metadata.setter
620 def metadata(self, metadata):
621 """
622 Set the metadata of the SR.
623 :param dict metadata: Dictionary that contains metadata.
624 """
626 assert isinstance(metadata, dict)
627 sr_properties = self._get_sr_properties()
628 sr_properties[self.PROP_METADATA] = json.dumps(metadata)
630 @property
631 def disconnected_hosts(self):
632 """
633 Get the list of disconnected hosts.
634 :return: Set that contains disconnected hosts.
635 :rtype: set(str)
636 """
638 disconnected_hosts = set()
639 for pool in self._get_storage_pools():
640 for report in pool.reports:
641 if report.ret_code & linstor.consts.WARN_NOT_CONNECTED == \
642 linstor.consts.WARN_NOT_CONNECTED:
643 disconnected_hosts.add(pool.node_name)
644 break
645 return disconnected_hosts
647 def check_volume_exists(self, volume_uuid):
648 """
649 Check if a volume exists in the SR.
650 :return: True if volume exists.
651 :rtype: bool
652 """
653 return volume_uuid in self._volumes
655 def create_volume(
656 self,
657 volume_uuid,
658 size,
659 persistent=True,
660 volume_name=None,
661 high_availability=False
662 ):
663 """
664 Create a new volume on the SR.
665 :param str volume_uuid: The volume uuid to use.
666 :param int size: volume size in B.
667 :param bool persistent: If false the volume will be unavailable
668 on the next constructor call LinstorSR(...).
669 :param str volume_name: If set, this name is used in the LINSTOR
670 database instead of a generated name.
671 :param bool high_availability: If set, the volume is created in
672 the HA group.
673 :return: The current device path of the volume.
674 :rtype: str
675 """
677 self._logger('Creating LINSTOR volume {}...'.format(volume_uuid))
678 if not volume_name:
679 volume_name = self.build_volume_name(util.gen_uuid())
680 volume_properties = self._create_volume_with_properties(
681 volume_uuid,
682 volume_name,
683 size,
684 True, # place_resources
685 high_availability
686 )
688 # Volume created! Now try to find the device path.
689 try:
690 self._logger(
691 'Find device path of LINSTOR volume {}...'.format(volume_uuid)
692 )
693 device_path = self._find_device_path(volume_uuid, volume_name)
694 if persistent:
695 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_EXISTS
696 self._volumes.add(volume_uuid)
697 self._logger(
698 'LINSTOR volume {} created!'.format(volume_uuid)
699 )
700 return device_path
701 except Exception:
702 # There is an issue to find the path.
703 # At this point the volume has just been created, so force flag can be used.
704 self._destroy_volume(volume_uuid, force=True)
705 raise
707 def mark_volume_as_persistent(self, volume_uuid):
708 """
709 Mark volume as persistent if created with persistent=False.
710 :param str volume_uuid: The volume uuid to mark.
711 """
713 self._ensure_volume_exists(volume_uuid)
715 # Mark volume as persistent.
716 volume_properties = self._get_volume_properties(volume_uuid)
717 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_EXISTS
719 def destroy_volume(self, volume_uuid):
720 """
721 Destroy a volume.
722 :param str volume_uuid: The volume uuid to destroy.
723 """
725 self._ensure_volume_exists(volume_uuid)
726 self.ensure_volume_is_not_locked(volume_uuid)
728 is_volume_in_use = any(node["in-use"] for node in self.get_resource_info(volume_uuid)["nodes"].values())
729 if is_volume_in_use:
730 raise LinstorVolumeManagerError(
731 f"Could not destroy volume `{volume_uuid}` as it is currently in use",
732 LinstorVolumeManagerError.ERR_VOLUME_IN_USE
733 )
735 # Mark volume as destroyed.
736 volume_properties = self._get_volume_properties(volume_uuid)
737 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_NOT_EXISTS
739 try:
740 self._volumes.remove(volume_uuid)
741 self._destroy_volume(volume_uuid)
742 except Exception as e:
743 raise LinstorVolumeManagerError(
744 str(e),
745 LinstorVolumeManagerError.ERR_VOLUME_DESTROY
746 )
748 def lock_volume(self, volume_uuid, locked=True):
749 """
750 Prevent modifications of the volume properties during
751 "self.LOCKED_EXPIRATION_DELAY" seconds. The SR must be locked
752 when used. This method is useful to attach/detach correctly a volume on
753 a slave. Without it the GC can rename a volume, in this case the old
754 volume path can be used by a slave...
755 :param str volume_uuid: The volume uuid to protect/unprotect.
756 :param bool locked: Lock/unlock the volume.
757 """
759 self._ensure_volume_exists(volume_uuid)
761 self._logger(
762 '{} volume {} as locked'.format(
763 'Mark' if locked else 'Unmark',
764 volume_uuid
765 )
766 )
768 volume_properties = self._get_volume_properties(volume_uuid)
769 if locked:
770 volume_properties[
771 self.PROP_IS_READONLY_TIMESTAMP
772 ] = str(time.time())
773 elif self.PROP_IS_READONLY_TIMESTAMP in volume_properties:
774 volume_properties.pop(self.PROP_IS_READONLY_TIMESTAMP)
776 def ensure_volume_is_not_locked(self, volume_uuid, timeout=None):
777 """
778 Ensure a volume is not locked. Wait if necessary.
779 :param str volume_uuid: The volume uuid to check.
780 :param int timeout: If the volume is always locked after the expiration
781 of the timeout, an exception is thrown.
782 """
783 return self.ensure_volume_list_is_not_locked([volume_uuid], timeout)
785 def ensure_volume_list_is_not_locked(self, volume_uuids, timeout=None):
786 checked = set()
787 for volume_uuid in volume_uuids:
788 if volume_uuid in self._volumes:
789 checked.add(volume_uuid)
791 if not checked:
792 return
794 waiting = False
796 volume_properties = self._get_kv_cache()
798 start = time.time()
799 while True:
800 # Can't delete in for loop, use a copy of the list.
801 remaining = checked.copy()
802 for volume_uuid in checked:
803 volume_properties.namespace = \
804 self._build_volume_namespace(volume_uuid)
805 timestamp = volume_properties.get(
806 self.PROP_IS_READONLY_TIMESTAMP
807 )
808 if timestamp is None:
809 remaining.remove(volume_uuid)
810 continue
812 now = time.time()
813 if now - float(timestamp) > self.LOCKED_EXPIRATION_DELAY:
814 self._logger(
815 'Remove readonly timestamp on {}'.format(volume_uuid)
816 )
817 volume_properties.pop(self.PROP_IS_READONLY_TIMESTAMP)
818 remaining.remove(volume_uuid)
819 continue
821 if not waiting:
822 self._logger(
823 'Volume {} is locked, waiting...'.format(volume_uuid)
824 )
825 waiting = True
826 break
828 if not remaining:
829 break
830 checked = remaining
832 if timeout is not None and now - start > timeout:
833 raise LinstorVolumeManagerError(
834 'volume `{}` is locked and timeout has been reached'
835 .format(volume_uuid),
836 LinstorVolumeManagerError.ERR_VOLUME_NOT_EXISTS
837 )
839 # We must wait to use the volume. After that we can modify it
840 # ONLY if the SR is locked to avoid bad reads on the slaves.
841 time.sleep(1)
842 volume_properties = self._create_kv_cache()
844 if waiting:
845 self._logger('No volume locked now!')
847 def remove_volume_if_diskless(self, volume_uuid):
848 """
849 Remove disless path from local node.
850 :param str volume_uuid: The volume uuid to remove.
851 """
853 self._ensure_volume_exists(volume_uuid)
855 volume_properties = self._get_volume_properties(volume_uuid)
856 volume_name = volume_properties.get(self.PROP_VOLUME_NAME)
858 node_name = socket.gethostname()
860 for resource in self._get_resource_cache().resources:
861 if resource.name == volume_name and resource.node_name == node_name:
862 if linstor.consts.FLAG_TIE_BREAKER in resource.flags:
863 return
864 break
866 if has_local_volume_openers(volume_name, 0):
867 return
869 result = self._linstor.resource_delete_if_diskless(
870 node_name=node_name, rsc_name=volume_name
871 )
872 if not linstor.Linstor.all_api_responses_no_error(result):
873 raise LinstorVolumeManagerError(
874 'Unable to delete diskless path of `{}` on node `{}`: {}'
875 .format(volume_name, node_name, ', '.join(
876 [str(x) for x in result]))
877 )
879 def introduce_volume(self, volume_uuid):
880 pass # TODO: Implement me.
882 def resize_volume(self, volume_uuid, new_size):
883 """
884 Resize a volume.
885 :param str volume_uuid: The volume uuid to resize.
886 :param int new_size: New size in B.
887 """
889 volume_name = self.get_volume_name(volume_uuid)
890 self.ensure_volume_is_not_locked(volume_uuid)
891 new_size = self.round_up_volume_size(new_size) // 1024
893 # We can't resize anything until DRBD is up to date.
894 # We wait here for 5min max and raise an easy to understand error for the user.
895 # 5min is an arbitrary time, it's impossible to get a fit all situation value
896 # and it's currently impossible to know how much time we have to wait
897 # This is mostly an issue for thick provisioning, thin isn't affected.
898 start_time = time.monotonic()
899 try:
900 self._linstor.resource_dfn_wait_synced(volume_name, wait_interval=1.0, timeout=60*5)
901 except linstor.LinstorTimeoutError:
902 raise LinstorVolumeManagerError(
903 f"Volume resizing of `{volume_uuid}` from SR `{self._group_name}` is incomplete: timeout reached but it continues in background."
904 )
905 util.SMlog(f"DRBD is up to date, syncing took {time.monotonic() - start_time}s")
907 result = self._linstor.volume_dfn_modify(
908 rsc_name=volume_name,
909 volume_nr=0,
910 size=new_size
911 )
913 self._mark_resource_cache_as_dirty()
915 error_str = self._get_error_str(result)
916 if error_str:
917 raise LinstorVolumeManagerError(
918 f"Could not resize volume `{volume_uuid}` from SR `{self._group_name}`: {error_str}"
919 )
921 def get_volume_name(self, volume_uuid):
922 """
923 Get the name of a particular volume.
924 :param str volume_uuid: The volume uuid of the name to get.
925 :return: The volume name.
926 :rtype: str
927 """
929 self._ensure_volume_exists(volume_uuid)
930 volume_properties = self._get_volume_properties(volume_uuid)
931 volume_name = volume_properties.get(self.PROP_VOLUME_NAME)
932 if volume_name:
933 return volume_name
934 raise LinstorVolumeManagerError(
935 'Failed to get volume name of {}'.format(volume_uuid)
936 )
938 def get_volume_size(self, volume_uuid):
939 """
940 Get the size of a particular volume.
941 :param str volume_uuid: The volume uuid of the size to get.
942 :return: The volume size.
943 :rtype: int
944 """
946 volume_name = self.get_volume_name(volume_uuid)
947 dfns = self._linstor.resource_dfn_list_raise(
948 query_volume_definitions=True,
949 filter_by_resource_definitions=[volume_name]
950 ).resource_definitions
952 size = dfns[0].volume_definitions[0].size
953 if size < 0:
954 raise LinstorVolumeManagerError(
955 'Failed to get volume size of: {}'.format(volume_uuid)
956 )
957 return size * 1024
959 def set_auto_promote_timeout(self, volume_uuid, timeout):
960 """
961 Define the blocking time of open calls when a DRBD
962 is already open on another host.
963 :param str volume_uuid: The volume uuid to modify.
964 """
966 volume_name = self.get_volume_name(volume_uuid)
967 result = self._linstor.resource_dfn_modify(volume_name, {
968 'DrbdOptions/Resource/auto-promote-timeout': timeout
969 })
970 error_str = self._get_error_str(result)
971 if error_str:
972 raise LinstorVolumeManagerError(
973 'Could not change the auto promote timeout of `{}`: {}'
974 .format(volume_uuid, error_str)
975 )
977 def set_drbd_ha_properties(self, volume_name, enabled=True):
978 """
979 Set or not HA DRBD properties required by drbd-reactor and
980 by specific volumes.
981 :param str volume_name: The volume to modify.
982 :param bool enabled: Enable or disable HA properties.
983 """
985 properties = {
986 'DrbdOptions/auto-quorum': 'disabled',
987 'DrbdOptions/Resource/auto-promote': 'no',
988 'DrbdOptions/Resource/on-no-data-accessible': 'io-error',
989 'DrbdOptions/Resource/on-no-quorum': 'io-error',
990 'DrbdOptions/Resource/on-suspended-primary-outdated': 'force-secondary',
991 'DrbdOptions/Resource/quorum': 'majority'
992 }
993 if enabled:
994 result = self._linstor.resource_dfn_modify(volume_name, properties)
995 else:
996 result = self._linstor.resource_dfn_modify(volume_name, {}, delete_props=list(properties.keys()))
998 error_str = self._get_error_str(result)
999 if error_str:
1000 raise LinstorVolumeManagerError(
1001 'Could not modify HA DRBD properties on volume `{}`: {}'
1002 .format(volume_name, error_str)
1003 )
1005 def get_volume_info(self, volume_uuid):
1006 """
1007 Get the volume info of a particular volume.
1008 :param str volume_uuid: The volume uuid of the volume info to get.
1009 :return: The volume info.
1010 :rtype: VolumeInfo
1011 """
1013 volume_name = self.get_volume_name(volume_uuid)
1014 try:
1015 return self._get_volumes_info()[volume_name]
1016 except KeyError:
1017 raise LinstorVolumeManagerError(
1018 f"Could not find info about volume `{volume_uuid}`, the resource definition has likely been deleted",
1019 LinstorVolumeManagerError.ERR_VOLUME_NOT_EXISTS
1020 )
1022 def get_device_path(self, volume_uuid):
1023 """
1024 Get the dev path of a volume, create a diskless if necessary.
1025 :param str volume_uuid: The volume uuid to get the dev path.
1026 :return: The current device path of the volume.
1027 :rtype: str
1028 """
1030 volume_name = self.get_volume_name(volume_uuid)
1031 return self._find_device_path(volume_uuid, volume_name)
1033 def get_volume_uuid_from_device_path(self, device_path):
1034 """
1035 Get the volume uuid of a device_path.
1036 :param str device_path: The dev path to find the volume uuid.
1037 :return: The volume uuid of the local device path.
1038 :rtype: str
1039 """
1041 expected_volume_name = \
1042 self.get_volume_name_from_device_path(device_path)
1044 volume_names = self.get_volumes_with_name()
1045 for volume_uuid, volume_name in volume_names.items():
1046 if volume_name == expected_volume_name:
1047 return volume_uuid
1049 raise LinstorVolumeManagerError(
1050 'Unable to find volume uuid from dev path `{}`'.format(device_path)
1051 )
1053 def get_volume_name_from_device_path(self, device_path):
1054 """
1055 Get the volume name of a device_path.
1056 :param str device_path: The dev path to find the volume name.
1057 :return: The volume name of the device path.
1058 :rtype: str
1059 """
1061 # Assume that we have a path like this:
1062 # - "/dev/drbd/by-res/xcp-volume-<UUID>/0"
1063 # - "../xcp-volume-<UUID>/0"
1064 if device_path.startswith(DRBD_BY_RES_PATH):
1065 prefix_len = len(DRBD_BY_RES_PATH)
1066 elif device_path.startswith('../'):
1067 prefix_len = 3
1068 else:
1069 raise LinstorVolumeManagerError('Unexpected device path: `{}`'.format(device_path))
1071 res_name_end = device_path.find('/', prefix_len)
1072 assert res_name_end != -1
1073 return device_path[prefix_len:res_name_end]
1075 def update_volume_uuid(self, volume_uuid, new_volume_uuid, force=False):
1076 """
1077 Change the uuid of a volume.
1078 :param str volume_uuid: The volume to modify.
1079 :param str new_volume_uuid: The new volume uuid to use.
1080 :param bool force: If true we doesn't check if volume_uuid is in the
1081 volume list. I.e. the volume can be marked as deleted but the volume
1082 can still be in the LINSTOR KV store if the deletion has failed.
1083 In specific cases like "undo" after a failed clone we must rename a bad
1084 deleted VDI.
1085 """
1087 self._logger(
1088 'Trying to update volume UUID {} to {}...'
1089 .format(volume_uuid, new_volume_uuid)
1090 )
1091 assert volume_uuid != new_volume_uuid, 'can\'t update volume UUID, same value'
1093 if not force:
1094 self._ensure_volume_exists(volume_uuid)
1095 self.ensure_volume_is_not_locked(volume_uuid)
1097 if new_volume_uuid in self._volumes:
1098 raise LinstorVolumeManagerError(
1099 'Volume `{}` already exists'.format(new_volume_uuid),
1100 LinstorVolumeManagerError.ERR_VOLUME_EXISTS
1101 )
1103 volume_properties = self._get_volume_properties(volume_uuid)
1104 if volume_properties.get(self.PROP_UPDATING_UUID_SRC):
1105 raise LinstorVolumeManagerError(
1106 'Cannot update volume uuid {}: invalid state'
1107 .format(volume_uuid)
1108 )
1110 # 1. Copy in temp variables metadata and volume_name.
1111 metadata = volume_properties.get(self.PROP_METADATA)
1112 volume_name = volume_properties.get(self.PROP_VOLUME_NAME)
1114 # 2. Switch to new volume namespace.
1115 volume_properties.namespace = self._build_volume_namespace(
1116 new_volume_uuid
1117 )
1119 if list(volume_properties.items()):
1120 raise LinstorVolumeManagerError(
1121 'Cannot update volume uuid {} to {}: '
1122 .format(volume_uuid, new_volume_uuid) +
1123 'this last one is not empty'
1124 )
1126 try:
1127 # 3. Mark new volume properties with PROP_UPDATING_UUID_SRC.
1128 # If we crash after that, the new properties can be removed
1129 # properly.
1130 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_NOT_EXISTS
1131 volume_properties[self.PROP_UPDATING_UUID_SRC] = volume_uuid
1133 # 4. Copy the properties.
1134 # Note: On new volumes, during clone for example, the metadata
1135 # may be missing. So we must test it to avoid this error:
1136 # "None has to be a str/unicode, but is <type 'NoneType'>"
1137 if metadata:
1138 volume_properties[self.PROP_METADATA] = metadata
1139 volume_properties[self.PROP_VOLUME_NAME] = volume_name
1141 # 5. Ok!
1142 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_EXISTS
1143 except Exception as err:
1144 try:
1145 # Clear the new volume properties in case of failure.
1146 assert volume_properties.namespace == \
1147 self._build_volume_namespace(new_volume_uuid)
1148 volume_properties.clear()
1149 except Exception as e:
1150 self._logger(
1151 'Failed to clear new volume properties: {} (ignoring...)'
1152 .format(e)
1153 )
1154 raise LinstorVolumeManagerError(
1155 'Failed to copy volume properties: {}'.format(err)
1156 )
1158 try:
1159 # 6. After this point, it's ok we can remove the
1160 # PROP_UPDATING_UUID_SRC property and clear the src properties
1161 # without problems.
1163 # 7. Switch to old volume namespace.
1164 volume_properties.namespace = self._build_volume_namespace(
1165 volume_uuid
1166 )
1167 volume_properties.clear()
1169 # 8. Switch a last time to new volume namespace.
1170 volume_properties.namespace = self._build_volume_namespace(
1171 new_volume_uuid
1172 )
1173 volume_properties.pop(self.PROP_UPDATING_UUID_SRC)
1174 except Exception as e:
1175 raise LinstorVolumeManagerError(
1176 'Failed to clear volume properties '
1177 'after volume uuid update: {}'.format(e)
1178 )
1180 try:
1181 self._volumes.remove(volume_uuid)
1182 except KeyError:
1183 # Can be missing if we are building the volume set attr AND
1184 # we are processing a deleted resource.
1185 assert force
1187 self._volumes.add(new_volume_uuid)
1189 self._logger(
1190 'UUID update succeeded of {} to {}! (properties={})'
1191 .format(
1192 volume_uuid, new_volume_uuid,
1193 self._get_filtered_properties(volume_properties)
1194 )
1195 )
1197 def update_volume_name(self, volume_uuid, volume_name):
1198 """
1199 Change the volume name of a volume.
1200 :param str volume_uuid: The volume to modify.
1201 :param str volume_name: The volume_name to use.
1202 """
1204 self._ensure_volume_exists(volume_uuid)
1205 self.ensure_volume_is_not_locked(volume_uuid)
1206 if not volume_name.startswith(self.PREFIX_VOLUME):
1207 raise LinstorVolumeManagerError(
1208 'Volume name `{}` must be start with `{}`'
1209 .format(volume_name, self.PREFIX_VOLUME)
1210 )
1212 if volume_name not in self._fetch_resource_names():
1213 raise LinstorVolumeManagerError(
1214 'Volume `{}` doesn\'t exist'.format(volume_name)
1215 )
1217 volume_properties = self._get_volume_properties(volume_uuid)
1218 volume_properties[self.PROP_VOLUME_NAME] = volume_name
1220 def get_usage_states(self, volume_uuid):
1221 """
1222 Check if a volume is currently used.
1223 :param str volume_uuid: The volume uuid to check.
1224 :return: A dictionary that contains states.
1225 :rtype: dict(str, bool or None)
1226 """
1228 states = {}
1230 volume_name = self.get_volume_name(volume_uuid)
1231 for resource_state in self._linstor.resource_list_raise(
1232 filter_by_resources=[volume_name]
1233 ).resource_states:
1234 states[resource_state.node_name] = resource_state.in_use
1236 return states
1238 def get_volume_openers(self, volume_uuid) -> LinstorVolumeOpeners:
1239 """
1240 Get openers of a volume.
1241 :param str volume_uuid: The volume uuid to monitor.
1242 :return: A dictionary that contains openers.
1243 :rtype: dict(str, obj)
1244 """
1245 return get_all_volume_openers(self.get_volume_name(volume_uuid), '0')
1247 def get_volumes_with_name(self):
1248 """
1249 Give a volume dictionary that contains names actually owned.
1250 :return: A volume/name dict.
1251 :rtype: dict(str, str)
1252 """
1253 return self._get_volumes_by_property(self.REG_VOLUME_NAME)
1255 def get_volumes_with_info(self):
1256 """
1257 Give a volume dictionary that contains VolumeInfos.
1258 :return: A volume/VolumeInfo dict.
1259 :rtype: dict(str, VolumeInfo)
1260 """
1262 volumes = {}
1264 volume_names = self.get_volumes_with_name()
1265 all_volume_info = self._get_volumes_info(volume_names)
1266 for volume_uuid, volume_name in volume_names.items():
1267 if volume_name:
1268 volume_info = all_volume_info.get(volume_name)
1269 if volume_info:
1270 volumes[volume_uuid] = volume_info
1271 continue
1273 # Well I suppose if this volume is not available,
1274 # LINSTOR has been used directly without using this API.
1275 volumes[volume_uuid] = self.VolumeInfo('')
1277 return volumes
1279 def get_volumes_with_metadata(self):
1280 """
1281 Give a volume dictionary that contains metadata.
1282 :return: A volume/metadata dict.
1283 :rtype: dict(str, dict)
1284 """
1286 volumes = {}
1288 metadata = self._get_volumes_by_property(self.REG_METADATA)
1289 for volume_uuid, volume_metadata in metadata.items():
1290 if volume_metadata:
1291 volume_metadata = json.loads(volume_metadata)
1292 if isinstance(volume_metadata, dict):
1293 volumes[volume_uuid] = volume_metadata
1294 continue
1295 raise LinstorVolumeManagerError(
1296 'Expected dictionary in volume metadata: {}'
1297 .format(volume_uuid)
1298 )
1300 volumes[volume_uuid] = {}
1302 return volumes
1304 def get_volume_metadata(self, volume_uuid):
1305 """
1306 Get the metadata of a volume.
1307 :return: Dictionary that contains metadata.
1308 :rtype: dict
1309 """
1311 self._ensure_volume_exists(volume_uuid)
1312 volume_properties = self._get_volume_properties(volume_uuid)
1313 metadata = volume_properties.get(self.PROP_METADATA)
1314 if metadata:
1315 metadata = json.loads(metadata)
1316 if isinstance(metadata, dict):
1317 return metadata
1318 raise LinstorVolumeManagerError(
1319 'Expected dictionary in volume metadata: {}'
1320 .format(volume_uuid)
1321 )
1322 return {}
1324 def set_volume_metadata(self, volume_uuid, metadata):
1325 """
1326 Set the metadata of a volume.
1327 :param dict metadata: Dictionary that contains metadata.
1328 """
1330 self._ensure_volume_exists(volume_uuid)
1331 self.ensure_volume_is_not_locked(volume_uuid)
1333 assert isinstance(metadata, dict)
1334 volume_properties = self._get_volume_properties(volume_uuid)
1335 volume_properties[self.PROP_METADATA] = json.dumps(metadata)
1337 def update_volume_metadata(self, volume_uuid, metadata):
1338 """
1339 Update the metadata of a volume. It modify only the given keys.
1340 It doesn't remove unreferenced key instead of set_volume_metadata.
1341 :param dict metadata: Dictionary that contains metadata.
1342 """
1344 self._ensure_volume_exists(volume_uuid)
1345 self.ensure_volume_is_not_locked(volume_uuid)
1347 assert isinstance(metadata, dict)
1348 volume_properties = self._get_volume_properties(volume_uuid)
1350 current_metadata = json.loads(
1351 volume_properties.get(self.PROP_METADATA, '{}')
1352 )
1353 if not isinstance(metadata, dict):
1354 raise LinstorVolumeManagerError(
1355 'Expected dictionary in volume metadata: {}'
1356 .format(volume_uuid)
1357 )
1359 for key, value in metadata.items():
1360 current_metadata[key] = value
1361 volume_properties[self.PROP_METADATA] = json.dumps(current_metadata)
1363 def shallow_clone_volume(self, volume_uuid, clone_uuid, persistent=True):
1364 """
1365 Clone a volume. Do not copy the data, this method creates a new volume
1366 with the same size.
1367 :param str volume_uuid: The volume to clone.
1368 :param str clone_uuid: The cloned volume.
1369 :param bool persistent: If false the volume will be unavailable
1370 on the next constructor call LinstorSR(...).
1371 :return: The current device path of the cloned volume.
1372 :rtype: str
1373 """
1375 volume_name = self.get_volume_name(volume_uuid)
1376 self.ensure_volume_is_not_locked(volume_uuid)
1378 # 1. Find ideal nodes + size to use.
1379 ideal_node_names, size = self._get_volume_node_names_and_size(
1380 volume_name
1381 )
1382 if size <= 0:
1383 raise LinstorVolumeManagerError(
1384 'Invalid size of {} for volume `{}`'.format(size, volume_name)
1385 )
1387 # 2. Create clone!
1388 return self.create_volume(clone_uuid, size, persistent)
1390 def remove_resourceless_volumes(self):
1391 """
1392 Remove all volumes without valid or non-empty name
1393 (i.e. without LINSTOR resource). It's different than
1394 LinstorVolumeManager constructor that takes a `repair` param that
1395 removes volumes with `PROP_NOT_EXISTS` to 1.
1396 """
1398 resource_names = self._fetch_resource_names()
1399 for volume_uuid, volume_name in self.get_volumes_with_name().items():
1400 if not volume_name or volume_name not in resource_names:
1401 # Don't force, we can be sure of what's happening.
1402 self.destroy_volume(volume_uuid)
1404 def destroy(self):
1405 """
1406 Destroy this SR. Object should not be used after that.
1407 :param bool force: Try to destroy volumes before if true.
1408 """
1410 # 1. Ensure volume list is empty. No cost.
1411 if self._volumes:
1412 raise LinstorVolumeManagerError(
1413 'Cannot destroy LINSTOR volume manager: '
1414 'It exists remaining volumes'
1415 )
1417 # 2. Fetch ALL resource names.
1418 # This list may therefore contain volumes created outside
1419 # the scope of the driver.
1420 resource_names = self._fetch_resource_names(ignore_deleted=False)
1421 try:
1422 resource_names.remove(DATABASE_VOLUME_NAME)
1423 except KeyError:
1424 # Really strange to reach that point.
1425 # Normally we always have the database volume in the list.
1426 pass
1428 # 3. Ensure the resource name list is entirely empty...
1429 if resource_names:
1430 raise LinstorVolumeManagerError(
1431 'Cannot destroy LINSTOR volume manager: '
1432 'It exists remaining volumes (created externally or being deleted)'
1433 )
1435 # 4. Destroying...
1436 controller_is_running = self._controller_is_running()
1437 uri = 'linstor://localhost'
1438 try:
1439 if controller_is_running:
1440 self._start_controller(start=False)
1442 # 4.1. Umount LINSTOR database.
1443 self._mount_database_volume(
1444 self.build_device_path(DATABASE_VOLUME_NAME),
1445 mount=False,
1446 force=True
1447 )
1449 # 4.2. Refresh instance.
1450 self._start_controller(start=True)
1451 self._linstor = self._create_linstor_instance(
1452 uri, keep_uri_unmodified=True
1453 )
1455 # 4.3. Destroy database volume.
1456 self._destroy_resource(DATABASE_VOLUME_NAME)
1458 # 4.4. Refresh linstor connection.
1459 # Without we get this error:
1460 # "Cannot delete resource group 'xcp-sr-linstor_group_thin_device' because it has existing resource definitions.."
1461 # Because the deletion of the databse was not seen by Linstor for some reason.
1462 # It seems a simple refresh of the Linstor connection make it aware of the deletion.
1463 self._linstor.disconnect()
1464 self._linstor.connect()
1466 # 4.5. Destroy remaining drbd nodes on hosts.
1467 # We check if there is a DRBD node on hosts that could mean blocking when destroying resource groups.
1468 # It needs to be done locally by each host so we go through the linstor-manager plugin.
1469 # If we don't do this sometimes, the destroy will fail when trying to destroy the resource groups with:
1470 # "linstor-manager:destroy error: Failed to destroy SP `xcp-sr-linstor_group_thin_device` on node `r620-s2`: The specified storage pool 'xcp-sr-linstor_group_thin_device' on node 'r620-s2' can not be deleted as volumes / snapshot-volumes are still using it."
1471 session = util.timeout(5, util.ApiSession, "SM-linstor-destroy").session
1472 for host_ref in session.xenapi.host.get_all():
1473 try:
1474 response = session.xenapi.host.call_plugin(
1475 host_ref, 'linstor-manager', 'destroyDrbdVolumes', {'volume_group': self._group_name}
1476 )
1477 except Exception as e:
1478 util.SMlog('Calling destroyDrbdVolumes on host {} failed with error {}'.format(host_ref, e))
1480 # 4.6. Destroy group and storage pools.
1481 self._destroy_resource_group(self._linstor, self._group_name)
1482 self._destroy_resource_group(self._linstor, self._ha_group_name)
1483 for pool in self._get_storage_pools(force=True):
1484 self._destroy_storage_pool(
1485 self._linstor, pool.name, pool.node_name
1486 )
1487 except Exception as e:
1488 self._start_controller(start=controller_is_running)
1489 raise e
1491 try:
1492 self._start_controller(start=False)
1493 for file in os.listdir(DATABASE_PATH):
1494 if file != 'lost+found':
1495 os.remove(DATABASE_PATH + '/' + file)
1496 except Exception as e:
1497 util.SMlog(
1498 'Ignoring failure after LINSTOR SR destruction: {}'
1499 .format(e)
1500 )
1502 def find_up_to_date_diskful_nodes(self, volume_uuid):
1503 """
1504 Find all nodes that contain a specific volume using diskful disks.
1505 The disk must be up to data to be used.
1506 :param str volume_uuid: The volume to use.
1507 :return: The available nodes.
1508 :rtype: tuple(set(str), str)
1509 """
1511 volume_name = self.get_volume_name(volume_uuid)
1513 in_use_by = None
1514 node_names = set()
1516 resource_states = filter(
1517 lambda resource_state: resource_state.name == volume_name,
1518 self._get_resource_cache().resource_states
1519 )
1521 for resource_state in resource_states:
1522 volume_state = resource_state.volume_states[0]
1523 if volume_state.disk_state == 'UpToDate':
1524 node_names.add(resource_state.node_name)
1525 if resource_state.in_use:
1526 in_use_by = resource_state.node_name
1528 return (node_names, in_use_by)
1530 def invalidate_resource_cache(self):
1531 """
1532 If resources are impacted by external commands like vhdutil,
1533 it's necessary to call this function to invalidate current resource
1534 cache.
1535 """
1536 self._mark_resource_cache_as_dirty()
1538 def has_node(self, node_name):
1539 """
1540 Check if a node exists in the LINSTOR database.
1541 :rtype: bool
1542 """
1543 result = self._linstor.node_list()
1544 error_str = self._get_error_str(result)
1545 if error_str:
1546 raise LinstorVolumeManagerError(
1547 'Failed to list nodes using `{}`: {}'
1548 .format(node_name, error_str)
1549 )
1550 return bool(result[0].node(node_name))
1552 def create_node(self, node_name, ip):
1553 """
1554 Create a new node in the LINSTOR database.
1555 :param str node_name: Node name to use.
1556 :param str ip: Host IP to communicate.
1557 """
1558 result = self._linstor.node_create(
1559 node_name,
1560 linstor.consts.VAL_NODE_TYPE_CMBD,
1561 ip
1562 )
1563 errors = self._filter_errors(result)
1564 if errors:
1565 error_str = self._get_error_str(errors)
1566 raise LinstorVolumeManagerError(
1567 'Failed to create node `{}`: {}'.format(node_name, error_str)
1568 )
1570 def destroy_node(self, node_name):
1571 """
1572 Destroy a node in the LINSTOR database.
1573 :param str node_name: Node name to remove.
1574 """
1575 result = self._linstor.node_delete(node_name)
1576 errors = self._filter_errors(result)
1577 if errors:
1578 error_str = self._get_error_str(errors)
1579 raise LinstorVolumeManagerError(
1580 'Failed to destroy node `{}`: {}'.format(node_name, error_str)
1581 )
1583 def create_node_interface(self, node_name, name, ip):
1584 """
1585 Create a new node interface in the LINSTOR database.
1586 :param str node_name: Node name of the interface to use.
1587 :param str name: Interface to create.
1588 :param str ip: IP of the interface.
1589 """
1590 result = self._linstor.netinterface_create(node_name, name, ip)
1591 errors = self._filter_errors(result)
1592 if errors:
1593 error_str = self._get_error_str(errors)
1594 raise LinstorVolumeManagerError(
1595 'Failed to create node interface on `{}`: {}'.format(node_name, error_str)
1596 )
1598 def destroy_node_interface(self, node_name, name):
1599 """
1600 Destroy a node interface in the LINSTOR database.
1601 :param str node_name: Node name of the interface to remove.
1602 :param str name: Interface to remove.
1603 """
1605 if name == 'default':
1606 raise LinstorVolumeManagerError(
1607 'Unable to delete the default interface of a node!'
1608 )
1610 result = self._linstor.netinterface_delete(node_name, name)
1611 errors = self._filter_errors(result)
1612 if errors:
1613 error_str = self._get_error_str(errors)
1614 raise LinstorVolumeManagerError(
1615 'Failed to destroy node interface on `{}`: {}'.format(node_name, error_str)
1616 )
1618 def modify_node_interface(self, node_name, name, ip):
1619 """
1620 Modify a node interface in the LINSTOR database. Create it if necessary.
1621 :param str node_name: Node name of the interface to use.
1622 :param str name: Interface to modify or create.
1623 :param str ip: IP of the interface.
1624 """
1625 result = self._linstor.netinterface_create(node_name, name, ip)
1626 errors = self._filter_errors(result)
1627 if not errors:
1628 return
1630 if self._check_errors(errors, [linstor.consts.FAIL_EXISTS_NET_IF]):
1631 result = self._linstor.netinterface_modify(node_name, name, ip)
1632 errors = self._filter_errors(result)
1633 if not errors:
1634 return
1636 error_str = self._get_error_str(errors)
1637 raise LinstorVolumeManagerError(
1638 'Unable to modify interface on `{}`: {}'.format(node_name, error_str)
1639 )
1641 def list_node_interfaces(self, node_name):
1642 """
1643 List all node interfaces.
1644 :param str node_name: Node name to use to list interfaces.
1645 :rtype: list
1646 :
1647 """
1648 result = self._linstor.net_interface_list(node_name)
1649 if not result:
1650 raise LinstorVolumeManagerError(
1651 'Unable to list interfaces on `{}`: no list received'.format(node_name)
1652 )
1654 interfaces = {}
1655 for interface in result:
1656 interface = interface._rest_data
1657 interfaces[interface['name']] = {
1658 'address': interface['address'],
1659 'active': interface['is_active']
1660 }
1661 return interfaces
1663 def get_node_preferred_interface(self, node_name):
1664 """
1665 Get the preferred interface used by a node.
1666 :param str node_name: Node name of the interface to get.
1667 :rtype: str
1668 """
1669 try:
1670 nodes = self._linstor.node_list_raise([node_name]).nodes
1671 if nodes:
1672 properties = nodes[0].props
1673 return properties.get('PrefNic', 'default')
1674 raise LinstorVolumeManagerError('Node `{}` not found'.format(node_name))
1675 except Exception as e:
1676 raise LinstorVolumeManagerError(
1677 'Failed to get preferred interface: `{}`'.format(e)
1678 )
1680 def set_node_preferred_interface(self, node_name, name):
1681 """
1682 Set the preferred interface to use on a node.
1683 :param str node_name: Node name of the interface.
1684 :param str name: Preferred interface to use.
1685 """
1686 result = self._linstor.node_modify(node_name, property_dict={'PrefNic': name})
1687 errors = self._filter_errors(result)
1688 if errors:
1689 error_str = self._get_error_str(errors)
1690 raise LinstorVolumeManagerError(
1691 'Failed to set preferred node interface on `{}`: {}'.format(node_name, error_str)
1692 )
1694 def get_nodes_info(self):
1695 """
1696 Get all nodes + statuses, used or not by the pool.
1697 :rtype: dict(str, dict)
1698 """
1699 try:
1700 nodes = {}
1701 for node in self._linstor.node_list_raise().nodes:
1702 nodes[node.name] = node.connection_status
1703 return nodes
1704 except Exception as e:
1705 raise LinstorVolumeManagerError(
1706 'Failed to get all nodes: `{}`'.format(e)
1707 )
1709 def get_storage_pools_info(self):
1710 """
1711 Give all storage pools of current group name.
1712 :rtype: dict(str, list)
1713 """
1714 storage_pools = {}
1715 for pool in self._get_storage_pools(force=True):
1716 if pool.node_name not in storage_pools:
1717 storage_pools[pool.node_name] = []
1719 size = -1
1720 capacity = -1
1722 space = pool.free_space
1723 if space:
1724 size = space.free_capacity
1725 if size < 0:
1726 size = -1
1727 else:
1728 size *= 1024
1729 capacity = space.total_capacity
1730 if capacity <= 0:
1731 capacity = -1
1732 else:
1733 capacity *= 1024
1735 storage_pools[pool.node_name].append({
1736 'name': pool.name,
1737 'linstor-uuid': pool.uuid,
1738 'free-size': size,
1739 'capacity': capacity
1740 })
1742 return storage_pools
1744 def get_resources_info(self):
1745 """
1746 Give all resources of current group name.
1747 :rtype: dict(str, list)
1748 """
1749 if self._resources_info_cache and not self._resource_cache_dirty:
1750 return self._resources_info_cache
1752 resources = {}
1753 resource_list = self._get_resource_cache()
1754 volume_names = self.get_volumes_with_name()
1755 for resource in resource_list.resources:
1756 if resource.name not in resources:
1757 resources[resource.name] = { 'nodes': {}, 'uuid': '' }
1758 resource_nodes = resources[resource.name]['nodes']
1760 resource_nodes[resource.node_name] = {
1761 'volumes': [],
1762 'diskful': linstor.consts.FLAG_DISKLESS not in resource.flags,
1763 'tie-breaker': linstor.consts.FLAG_TIE_BREAKER in resource.flags
1764 }
1765 resource_volumes = resource_nodes[resource.node_name]['volumes']
1767 for volume in resource.volumes:
1768 # We ignore diskless pools of the form "DfltDisklessStorPool".
1769 if volume.storage_pool_name != self._group_name:
1770 continue
1772 usable_size = volume.usable_size
1773 if usable_size < 0:
1774 usable_size = -1
1775 else:
1776 usable_size *= 1024
1778 allocated_size = volume.allocated_size
1779 if allocated_size < 0:
1780 allocated_size = -1
1781 else:
1782 allocated_size *= 1024
1784 resource_volumes.append({
1785 'storage-pool-name': volume.storage_pool_name,
1786 'linstor-uuid': volume.uuid,
1787 'number': volume.number,
1788 'device-path': volume.device_path,
1789 'usable-size': usable_size,
1790 'allocated-size': allocated_size
1791 })
1793 for resource_state in resource_list.resource_states:
1794 resource = resources[resource_state.rsc_name]['nodes'][resource_state.node_name]
1795 resource['in-use'] = resource_state.in_use
1797 volumes = resource['volumes']
1798 for volume_state in resource_state.volume_states:
1799 volume = next((x for x in volumes if x['number'] == volume_state.number), None)
1800 if volume:
1801 volume['disk-state'] = volume_state.disk_state
1803 for volume_uuid, volume_name in volume_names.items():
1804 resource = resources.get(volume_name)
1805 if resource:
1806 resource['uuid'] = volume_uuid
1808 self._resources_info_cache = resources
1809 return self._resources_info_cache
1811 def get_resource_info(self, volume_uuid: str) -> Dict[str, Any]:
1812 """
1813 Give a resource info based on its UUID.
1814 :param volume_uuid str: volume uuid to search for
1815 :rtype: dict(str, any)
1816 """
1817 for volume in self.get_resources_info().values():
1818 if volume["uuid"] == volume_uuid:
1819 return volume
1821 raise LinstorVolumeManagerError(
1822 f"Could not find info about volume `{volume_uuid}`",
1823 LinstorVolumeManagerError.ERR_VOLUME_NOT_EXISTS
1824 )
1826 def get_database_path(self):
1827 """
1828 Get the database path.
1829 :return: The current database path.
1830 :rtype: str
1831 """
1832 return self._request_database_path(self._linstor, activate=True)
1834 @classmethod
1835 def is_controller(cls):
1836 return cls._is_mounted(DATABASE_PATH)
1838 @classmethod
1839 def get_database_backup_age(cls):
1840 """
1841 Return the latest backup age in seconds.
1842 If not called on the Controller, since backups are not available,
1843 returns a huge value (a timestamp of now).
1844 """
1845 return (datetime.now() - cls._get_latest_database_backup()[1]).total_seconds()
1847 def database_backup(self, name=""):
1848 # Create new backup
1849 date = datetime.now().strftime(DATABASE_BACKUP_DATE_FORMAT)
1850 filename = DATABASE_BACKUP_NAME_FORMAT.format(date, name)
1851 self._linstor.controller_backupdb(filename)
1852 # Relative path are ok for a secondary backup filename:
1853 # https://github.com/LINBIT/linstor-server/blob/3e9306a9d8215606544c64c50ced150625ee4926/controller/src/main/java/com/linbit/linstor/api/rest/v1/Controller.java#L408
1854 self._linstor.controller_backupdb(f"../linstor.d/db-backups/{filename}")
1855 util.SMlog(f"[database_backup] Created: {filename}", priority=util.LOG_INFO)
1857 @classmethod
1858 def database_backup_validate_and_prune(cls):
1859 """
1860 Removes old backup based on two criterias:
1861 - Validity of the zipfile done by self._check_database_backup.
1862 - Number of valid files found, only the nth latest are kept.
1863 """
1864 for directory in (DATABASE_BACKUP_DIR_MAIN, DATABASE_BACKUP_DIR_SPARE):
1865 valid_backup_count = 0
1866 # Validate file and apply retention
1867 for database_backup_path, _ in cls._get_sorted_database_backup(directory):
1868 try:
1869 cls._check_database_backup(database_backup_path)
1870 valid_backup_count += 1
1871 if valid_backup_count < DATABASE_BACKUP_RETENTION:
1872 continue
1873 except LinstorDatabaseBackupError as error:
1874 util.SMlog(f"[database_backup] Check failed `{error}` [{database_backup_path}]",
1875 priority=util.LOG_ERR)
1876 with contextlib.suppress(OSError):
1877 os.unlink(database_backup_path)
1879 @classmethod
1880 def get_all_group_names(cls, base_name):
1881 """
1882 Get all group names. I.e. list of current group + HA.
1883 :param str base_name: The SR group_name to use.
1884 :return: List of group names.
1885 :rtype: list
1886 """
1887 return [cls.build_group_name(base_name), cls._build_ha_group_name(base_name)]
1889 @classmethod
1890 def get_volume_group_name(cls, volume_name) -> str:
1891 """
1892 Get the group name associated with a volume.
1893 :param str volume_name: The volume name to find.
1894 :return: A group name.
1895 :rtype: str
1896 """
1898 lin = cls._create_linstor_instance(uri=None)
1899 try:
1900 for dfn in lin.resource_dfn_list_raise(
1901 query_volume_definitions=False,
1902 filter_by_resource_definitions=[volume_name]
1903 ).resource_definitions:
1904 return dfn.resource_group_name
1905 finally:
1906 lin.disconnect()
1908 return ''
1910 @classmethod
1911 def create_sr(cls, group_name, ips, redundancy, thin_provisioning, logger=default_logger.__func__):
1912 """
1913 Create a new SR on the given nodes.
1914 :param str group_name: The SR group_name to use.
1915 :param set(str) ips: Node ips.
1916 :param int redundancy: How many copy of volumes should we store?
1917 :param bool thin_provisioning: Use thin or thick provisioning.
1918 :param function logger: Function to log messages.
1919 :return: A new LinstorSr instance.
1920 :rtype: LinstorSr
1921 """
1923 try:
1924 cls._start_controller(start=True)
1925 sr = cls._create_sr(group_name, ips, redundancy, thin_provisioning, logger)
1926 finally:
1927 # Controller must be stopped and volume unmounted because
1928 # it is the role of the drbd-reactor daemon to do the right
1929 # actions.
1930 cls._start_controller(start=False)
1931 cls._mount_volume(
1932 cls.build_device_path(DATABASE_VOLUME_NAME),
1933 DATABASE_PATH,
1934 mount=False
1935 )
1936 return sr
1938 @classmethod
1939 def _create_sr(cls, group_name, ips, redundancy, thin_provisioning, logger=default_logger.__func__):
1940 # 1. Check if SR already exists.
1941 uri = 'linstor://localhost'
1943 lin = cls._create_linstor_instance(uri, keep_uri_unmodified=True)
1945 node_names = list(ips.keys())
1946 for node_name, ip in ips.items():
1947 while True:
1948 # Try to create node.
1949 result = lin.node_create(
1950 node_name,
1951 linstor.consts.VAL_NODE_TYPE_CMBD,
1952 ip
1953 )
1955 errors = cls._filter_errors(result)
1956 if cls._check_errors(
1957 errors, [linstor.consts.FAIL_EXISTS_NODE]
1958 ):
1959 # If it already exists, remove, then recreate.
1960 result = lin.node_delete(node_name)
1961 error_str = cls._get_error_str(result)
1962 if error_str:
1963 raise LinstorVolumeManagerError(
1964 'Failed to remove old node `{}`: {}'
1965 .format(node_name, error_str)
1966 )
1967 elif not errors:
1968 break # Created!
1969 else:
1970 raise LinstorVolumeManagerError(
1971 'Failed to create node `{}` with ip `{}`: {}'.format(
1972 node_name, ip, cls._get_error_str(errors)
1973 )
1974 )
1976 driver_pool_name = group_name
1977 base_group_name = group_name
1978 group_name = cls.build_group_name(group_name)
1979 storage_pool_name = group_name
1980 pools = lin.storage_pool_list_raise(filter_by_stor_pools=[storage_pool_name]).storage_pools
1981 if pools:
1982 existing_node_names = [pool.node_name for pool in pools]
1983 raise LinstorVolumeManagerError(
1984 'Unable to create SR `{}`. It already exists on node(s): {}'
1985 .format(group_name, existing_node_names)
1986 )
1988 if lin.resource_group_list_raise(
1989 cls.get_all_group_names(base_group_name)
1990 ).resource_groups:
1991 if not lin.resource_dfn_list_raise().resource_definitions:
1992 backup_path = cls._create_database_backup_path()
1993 logger(
1994 'Group name already exists `{}` without LVs. '
1995 'Ignoring and moving the config files in {}'.format(group_name, backup_path)
1996 )
1997 cls._move_files(DATABASE_PATH, backup_path)
1998 else:
1999 raise LinstorVolumeManagerError(
2000 'Unable to create SR `{}`: The group name already exists'
2001 .format(group_name)
2002 )
2004 if thin_provisioning:
2005 driver_pool_parts = driver_pool_name.split('/')
2006 if not len(driver_pool_parts) == 2:
2007 raise LinstorVolumeManagerError(
2008 'Invalid group name using thin provisioning. '
2009 'Expected format: \'VG/LV`\''
2010 )
2012 # 2. Create storage pool on each node + resource group.
2013 reg_volume_group_not_found = re.compile(
2014 ".*Volume group '.*' not found$"
2015 )
2017 i = 0
2018 try:
2019 # 2.a. Create storage pools.
2020 storage_pool_count = 0
2021 while i < len(node_names):
2022 node_name = node_names[i]
2024 result = lin.storage_pool_create(
2025 node_name=node_name,
2026 storage_pool_name=storage_pool_name,
2027 storage_driver='LVM_THIN' if thin_provisioning else 'LVM',
2028 driver_pool_name=driver_pool_name
2029 )
2031 errors = linstor.Linstor.filter_api_call_response_errors(
2032 result
2033 )
2034 if errors:
2035 if errors[0].is_error(
2036 linstor.consts.FAIL_STOR_POOL_CONFIGURATION_ERROR
2037 ) and reg_volume_group_not_found.match(errors[0].message):
2038 logger(
2039 'Volume group `{}` not found on `{}`. Ignoring...'
2040 .format(group_name, node_name)
2041 )
2042 cls._destroy_storage_pool(lin, storage_pool_name, node_name)
2043 else:
2044 error_str = cls._get_error_str(result)
2045 raise LinstorVolumeManagerError(
2046 'Could not create SP `{}` on node `{}`: {}'
2047 .format(group_name, node_name, error_str)
2048 )
2049 else:
2050 storage_pool_count += 1
2051 i += 1
2053 if not storage_pool_count:
2054 raise LinstorVolumeManagerError(
2055 'Unable to create SR `{}`: No VG group found'.format(
2056 group_name,
2057 )
2058 )
2060 # 2.b. Create resource groups.
2061 ha_group_name = cls._build_ha_group_name(base_group_name)
2062 cls._create_resource_group(
2063 lin,
2064 group_name,
2065 storage_pool_name,
2066 redundancy,
2067 True
2068 )
2069 cls._create_resource_group(
2070 lin,
2071 ha_group_name,
2072 storage_pool_name,
2073 3,
2074 True
2075 )
2077 # 3. Create the LINSTOR database volume and mount it.
2078 try:
2079 logger('Creating database volume...')
2080 volume_path = cls._create_database_volume(
2081 lin, ha_group_name, storage_pool_name, node_names, redundancy
2082 )
2083 except LinstorVolumeManagerError as e:
2084 if e.code != LinstorVolumeManagerError.ERR_VOLUME_EXISTS:
2085 logger('Destroying database volume after creation fail...')
2086 cls._force_destroy_database_volume(lin, group_name)
2087 raise
2089 try:
2090 logger('Mounting database volume...')
2092 # First we must disable the controller to move safely the
2093 # LINSTOR config.
2094 cls._start_controller(start=False)
2096 cls._mount_database_volume(volume_path)
2097 except Exception as e:
2098 # Ensure we are connected because controller has been
2099 # restarted during mount call.
2100 logger('Destroying database volume after mount fail...')
2102 try:
2103 cls._start_controller(start=True)
2104 except Exception:
2105 pass
2107 lin = cls._create_linstor_instance(
2108 uri, keep_uri_unmodified=True
2109 )
2110 cls._force_destroy_database_volume(lin, group_name)
2111 raise e
2113 cls._start_controller(start=True)
2114 lin = cls._create_linstor_instance(uri, keep_uri_unmodified=True)
2116 # 4. Remove storage pools/resource/volume group in the case of errors.
2117 except Exception as e:
2118 logger('Destroying resource group and storage pools after fail...')
2119 try:
2120 cls._destroy_resource_group(lin, group_name)
2121 cls._destroy_resource_group(lin, ha_group_name)
2122 except Exception as e2:
2123 logger('Failed to destroy resource group: {}'.format(e2))
2124 pass
2125 j = 0
2126 i = min(i, len(node_names) - 1)
2127 while j <= i:
2128 try:
2129 cls._destroy_storage_pool(lin, storage_pool_name, node_names[j])
2130 except Exception as e2:
2131 logger('Failed to destroy resource group: {}'.format(e2))
2132 pass
2133 j += 1
2134 raise e
2136 # 5. Return new instance.
2137 instance = cls.__new__(cls)
2138 instance._linstor = lin
2139 instance._logger = logger
2140 instance._redundancy = redundancy
2141 instance._base_group_name = base_group_name
2142 instance._group_name = group_name
2143 instance._volumes = set()
2144 instance._storage_pools_time = 0
2145 instance._kv_cache = instance._create_kv_cache()
2146 instance._resource_cache = None
2147 instance._resource_cache_dirty = True
2148 instance._volume_info_cache = None
2149 instance._volume_info_cache_dirty = True
2150 return instance
2152 @classmethod
2153 def build_device_path(cls, volume_name):
2154 """
2155 Build a device path given a volume name.
2156 :param str volume_name: The volume name to use.
2157 :return: A valid or not device path.
2158 :rtype: str
2159 """
2161 return '{}{}/0'.format(cls.DEV_ROOT_PATH, volume_name)
2163 @classmethod
2164 def build_volume_name(cls, base_name):
2165 """
2166 Build a volume name given a base name (i.e. a UUID).
2167 :param str base_name: The volume name to use.
2168 :return: A valid or not device path.
2169 :rtype: str
2170 """
2171 return '{}{}'.format(cls.PREFIX_VOLUME, base_name)
2173 @classmethod
2174 def round_up_volume_size(cls, volume_size):
2175 """
2176 Align volume size on higher multiple of BLOCK_SIZE.
2177 :param int volume_size: The volume size to align.
2178 :return: An aligned volume size.
2179 :rtype: int
2180 """
2181 return round_up(volume_size, cls.BLOCK_SIZE)
2183 @classmethod
2184 def round_down_volume_size(cls, volume_size):
2185 """
2186 Align volume size on lower multiple of BLOCK_SIZE.
2187 :param int volume_size: The volume size to align.
2188 :return: An aligned volume size.
2189 :rtype: int
2190 """
2191 return round_down(volume_size, cls.BLOCK_SIZE)
2193 # --------------------------------------------------------------------------
2194 # Private helpers.
2195 # --------------------------------------------------------------------------
2197 def _create_kv_cache(self):
2198 self._kv_cache = self._create_linstor_kv('/')
2199 self._kv_cache_dirty = False
2200 return self._kv_cache
2202 def _get_kv_cache(self):
2203 if self._kv_cache_dirty:
2204 self._kv_cache = self._create_kv_cache()
2205 return self._kv_cache
2207 def _create_resource_cache(self):
2208 self._resource_cache = self._linstor.resource_list_raise()
2209 self._resource_cache_dirty = False
2210 return self._resource_cache
2212 def _get_resource_cache(self):
2213 if self._resource_cache_dirty:
2214 self._resource_cache = self._create_resource_cache()
2215 return self._resource_cache
2217 def _mark_resource_cache_as_dirty(self):
2218 self._resource_cache_dirty = True
2219 self._volume_info_cache_dirty = True
2221 # --------------------------------------------------------------------------
2223 def _ensure_volume_exists(self, volume_uuid):
2224 if volume_uuid not in self._volumes:
2225 raise LinstorVolumeManagerError(
2226 'volume `{}` doesn\'t exist'.format(volume_uuid),
2227 LinstorVolumeManagerError.ERR_VOLUME_NOT_EXISTS
2228 )
2230 def _find_best_size_candidates(self):
2231 result = self._linstor.resource_group_qmvs(self._group_name)
2232 error_str = self._get_error_str(result)
2233 if error_str:
2234 raise LinstorVolumeManagerError(
2235 'Failed to get max volume size allowed of SR `{}`: {}'.format(
2236 self._group_name,
2237 error_str
2238 )
2239 )
2240 return result[0].candidates
2242 def _fetch_resource_names(self, ignore_deleted=True):
2243 resource_names = set()
2244 dfns = self._linstor.resource_dfn_list_raise().resource_definitions
2245 for dfn in dfns:
2246 if dfn.resource_group_name in self.get_all_group_names(self._base_group_name) and (
2247 ignore_deleted or
2248 linstor.consts.FLAG_DELETE not in dfn.flags
2249 ):
2250 resource_names.add(dfn.name)
2251 return resource_names
2253 def _get_volumes_info(self, volume_names=None):
2254 all_volume_info = {}
2256 if not self._volume_info_cache_dirty:
2257 return self._volume_info_cache
2259 # `volume_names` MUST contain all volumes registered in the KV store.
2260 # It can be provided to the function to avoid double fetching.
2261 if not volume_names:
2262 volume_names = self.get_volumes_with_name()
2263 volume_names = set(volume_names.values())
2265 def process_resource(resource):
2266 if resource.name not in all_volume_info:
2267 current = all_volume_info[resource.name] = self.VolumeInfo(
2268 resource.name
2269 )
2270 else:
2271 current = all_volume_info[resource.name]
2273 if linstor.consts.FLAG_DISKLESS not in resource.flags:
2274 current.diskful.append(resource.node_name)
2276 for volume in resource.volumes:
2277 # We ignore diskless pools of the form "DfltDisklessStorPool".
2278 if volume.storage_pool_name != self._group_name:
2279 continue
2280 # Only fetch first volume.
2281 if volume.number != 0:
2282 continue
2284 allocated_size = volume.allocated_size
2285 if allocated_size > current.allocated_size:
2286 current.allocated_size = allocated_size
2288 usable_size = volume.usable_size
2289 if usable_size > 0 and (
2290 usable_size < current.virtual_size or
2291 not current.virtual_size
2292 ):
2293 current.virtual_size = usable_size
2295 try:
2296 for resource in self._get_resource_cache().resources:
2297 if resource.name in volume_names:
2298 process_resource(resource)
2299 for volume in all_volume_info.values():
2300 if volume.allocated_size <= 0:
2301 raise LinstorVolumeManagerError('Failed to get allocated size of `{}`'.format(resource.name))
2303 if volume.virtual_size <= 0:
2304 raise LinstorVolumeManagerError('Failed to get usable size of `{}`'.format(volume.name))
2306 volume.allocated_size *= 1024
2307 volume.virtual_size *= 1024
2308 except LinstorVolumeManagerError:
2309 self._mark_resource_cache_as_dirty()
2310 raise
2312 self._volume_info_cache_dirty = False
2313 self._volume_info_cache = all_volume_info
2315 return all_volume_info
2317 def _get_volume_node_names_and_size(self, volume_name):
2318 node_names = set()
2319 size = -1
2320 for resource in self._linstor.resource_list_raise(
2321 filter_by_resources=[volume_name]
2322 ).resources:
2323 for volume in resource.volumes:
2324 # We ignore diskless pools of the form "DfltDisklessStorPool".
2325 if volume.storage_pool_name != self._group_name:
2326 continue
2328 node_names.add(resource.node_name)
2330 usable_size = volume.usable_size
2331 if usable_size <= 0:
2332 continue
2334 if size < 0:
2335 size = usable_size
2336 else:
2337 size = min(size, usable_size)
2339 if size <= 0:
2340 raise LinstorVolumeManagerError('Failed to get usable size of `{}`'.format(resource.name))
2342 return (node_names, size * 1024)
2344 def _compute_size(self, attr):
2345 capacity = 0
2346 for pool in self._get_storage_pools(force=True):
2347 space = pool.free_space
2348 if space:
2349 size = getattr(space, attr)
2350 if size < 0:
2351 raise LinstorVolumeManagerError(
2352 'Failed to get pool {} attr of `{}`'
2353 .format(attr, pool.node_name)
2354 )
2355 capacity += size
2356 return capacity * 1024
2358 def _get_node_names(self):
2359 node_names = set()
2360 for pool in self._get_storage_pools():
2361 node_names.add(pool.node_name)
2362 return node_names
2364 def _get_storage_pools(self, force=False):
2365 cur_time = time.time()
2366 elsaped_time = cur_time - self._storage_pools_time
2368 if force or elsaped_time >= self.STORAGE_POOLS_FETCH_INTERVAL:
2369 self._storage_pools = self._linstor.storage_pool_list_raise(
2370 filter_by_stor_pools=[self._group_name]
2371 ).storage_pools
2372 self._storage_pools_time = time.time()
2374 return self._storage_pools
2376 def _create_volume(
2377 self,
2378 volume_uuid,
2379 volume_name,
2380 size,
2381 place_resources,
2382 high_availability
2383 ):
2384 size = self.round_up_volume_size(size)
2385 self._mark_resource_cache_as_dirty()
2387 group_name = self._ha_group_name if high_availability else self._group_name
2388 def create_definition():
2389 first_attempt = True
2390 while True:
2391 try:
2392 self._check_volume_creation_errors(
2393 self._linstor.resource_group_spawn(
2394 rsc_grp_name=group_name,
2395 rsc_dfn_name=volume_name,
2396 vlm_sizes=['{}B'.format(size)],
2397 definitions_only=True
2398 ),
2399 volume_uuid,
2400 self._group_name
2401 )
2402 break
2403 except LinstorVolumeManagerError as e:
2404 if (
2405 not first_attempt or
2406 not high_availability or
2407 e.code != LinstorVolumeManagerError.ERR_GROUP_NOT_EXISTS
2408 ):
2409 raise
2411 first_attempt = False
2412 self._create_resource_group(
2413 self._linstor,
2414 group_name,
2415 self._group_name,
2416 3,
2417 True
2418 )
2420 self._configure_volume_peer_slots(self._linstor, volume_name)
2422 def clean():
2423 try:
2424 self._destroy_volume(volume_uuid, force=True, preserve_properties=True)
2425 except Exception as e:
2426 self._logger(
2427 'Unable to destroy volume {} after creation fail: {}'
2428 .format(volume_uuid, e)
2429 )
2431 def create():
2432 try:
2433 create_definition()
2434 if place_resources:
2435 # Basic case when we use the default redundancy of the group.
2436 self._check_volume_creation_errors(
2437 self._linstor.resource_auto_place(
2438 rsc_name=volume_name,
2439 place_count=self._redundancy,
2440 diskless_on_remaining=False
2441 ),
2442 volume_uuid,
2443 self._group_name
2444 )
2445 except LinstorVolumeManagerError as e:
2446 if e.code != LinstorVolumeManagerError.ERR_VOLUME_EXISTS:
2447 clean()
2448 raise
2449 except Exception:
2450 clean()
2451 raise
2453 util.retry(create, maxretry=5)
2455 def _create_volume_with_properties(
2456 self,
2457 volume_uuid,
2458 volume_name,
2459 size,
2460 place_resources,
2461 high_availability
2462 ):
2463 if self.check_volume_exists(volume_uuid):
2464 raise LinstorVolumeManagerError(
2465 'Could not create volume `{}` from SR `{}`, it already exists'
2466 .format(volume_uuid, self._group_name) + ' in properties',
2467 LinstorVolumeManagerError.ERR_VOLUME_EXISTS
2468 )
2470 if volume_name in self._fetch_resource_names():
2471 raise LinstorVolumeManagerError(
2472 'Could not create volume `{}` from SR `{}`, '.format(
2473 volume_uuid, self._group_name
2474 ) + 'resource of the same name already exists in LINSTOR'
2475 )
2477 # I am paranoid.
2478 volume_properties = self._get_volume_properties(volume_uuid)
2479 if (volume_properties.get(self.PROP_NOT_EXISTS) is not None):
2480 raise LinstorVolumeManagerError(
2481 'Could not create volume `{}`, '.format(volume_uuid) +
2482 'properties already exist'
2483 )
2485 try:
2486 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_CREATING
2487 volume_properties[self.PROP_VOLUME_NAME] = volume_name
2489 self._create_volume(
2490 volume_uuid,
2491 volume_name,
2492 size,
2493 place_resources,
2494 high_availability
2495 )
2497 assert volume_properties.namespace == \
2498 self._build_volume_namespace(volume_uuid)
2499 return volume_properties
2500 except LinstorVolumeManagerError as e:
2501 # Do not destroy existing resource!
2502 # In theory we can't get this error because we check this event
2503 # before the `self._create_volume` case.
2504 # It can only happen if the same volume uuid is used in the same
2505 # call in another host.
2506 if e.code != LinstorVolumeManagerError.ERR_VOLUME_EXISTS:
2507 self._destroy_volume(volume_uuid, force=True)
2508 raise
2510 def _find_device_path(self, volume_uuid, volume_name):
2511 current_device_path = self._request_device_path(
2512 volume_uuid, volume_name, activate=True
2513 )
2515 # We use realpath here to get the /dev/drbd<id> path instead of
2516 # /dev/drbd/by-res/<resource_name>.
2517 expected_device_path = self.build_device_path(volume_name)
2518 util.wait_for_path(expected_device_path, 5)
2520 device_realpath = os.path.realpath(expected_device_path)
2521 if current_device_path != device_realpath:
2522 raise LinstorVolumeManagerError(
2523 'Invalid path, current={}, expected={} (realpath={})'
2524 .format(
2525 current_device_path,
2526 expected_device_path,
2527 device_realpath
2528 )
2529 )
2530 return expected_device_path
2532 def _request_device_path(self, volume_uuid, volume_name, activate=False):
2533 node_name = socket.gethostname()
2535 resource = next(filter(
2536 lambda resource: resource.node_name == node_name and
2537 resource.name == volume_name,
2538 self._get_resource_cache().resources
2539 ), None)
2541 if not resource:
2542 if activate:
2543 self._mark_resource_cache_as_dirty()
2544 self._activate_device_path(
2545 self._linstor, node_name, volume_name
2546 )
2547 return self._request_device_path(volume_uuid, volume_name)
2548 raise LinstorVolumeManagerError(
2549 'Unable to get dev path for `{}`, no resource found but definition "seems" to exist'
2550 .format(volume_uuid)
2551 )
2553 # Contains a path of the /dev/drbd<id> form.
2554 device_path = resource.volumes[0].device_path
2555 if not device_path:
2556 raise LinstorVolumeManagerError('Empty dev path for `{}`!'.format(volume_uuid))
2557 return device_path
2559 def _destroy_resource(self, resource_name, force=False):
2560 result = self._linstor.resource_dfn_delete(resource_name)
2561 error_str = self._get_error_str(result)
2562 if not error_str:
2563 self._mark_resource_cache_as_dirty()
2564 return
2566 if not force:
2567 self._mark_resource_cache_as_dirty()
2568 raise LinstorVolumeManagerError(
2569 'Could not destroy resource `{}` from SR `{}`: {}'
2570 .format(resource_name, self._group_name, error_str)
2571 )
2573 # If force is used, ensure there is no opener.
2574 openers = get_all_volume_openers(resource_name, '0')
2575 for host_openers in openers.values():
2576 if host_openers:
2577 self._mark_resource_cache_as_dirty()
2578 raise LinstorVolumeManagerError(
2579 'Could not force destroy resource `{}` from SR `{}`: {} (openers=`{}`)'
2580 .format(resource_name, self._group_name, error_str, openers)
2581 )
2583 # Maybe the resource is blocked in primary mode. DRBD/LINSTOR issue?
2584 resource_states = filter(
2585 lambda resource_state: resource_state.name == resource_name,
2586 self._get_resource_cache().resource_states
2587 )
2589 # Mark only after computation of states.
2590 self._mark_resource_cache_as_dirty()
2592 for resource_state in resource_states:
2593 volume_state = resource_state.volume_states[0]
2594 if resource_state.in_use:
2595 demote_drbd_resource(resource_state.node_name, resource_name)
2596 break
2597 self._destroy_resource(resource_name)
2599 def _destroy_volume(self, volume_uuid, force=False, preserve_properties=False):
2600 volume_properties = self._get_volume_properties(volume_uuid)
2601 try:
2602 volume_name = volume_properties.get(self.PROP_VOLUME_NAME)
2603 if volume_name in self._fetch_resource_names():
2604 self._destroy_resource(volume_name, force)
2606 # Assume this call is atomic.
2607 if not preserve_properties:
2608 volume_properties.clear()
2609 except Exception as e:
2610 raise LinstorVolumeManagerError(
2611 'Cannot destroy volume `{}`: {}'.format(volume_uuid, e)
2612 )
2614 def _build_volumes(self, repair):
2615 properties = self._kv_cache
2616 resource_names = self._fetch_resource_names()
2618 self._volumes = set()
2620 updating_uuid_volumes = self._get_volumes_by_property(
2621 self.REG_UPDATING_UUID_SRC, ignore_inexisting_volumes=False
2622 )
2623 if updating_uuid_volumes and not repair:
2624 raise LinstorVolumeManagerError(
2625 'Cannot build LINSTOR volume list: '
2626 'It exists invalid "updating uuid volumes", repair is required',
2627 LinstorVolumeManagerError.ERR_REPAIR_REQUIRED
2628 )
2630 existing_volumes = self._get_volumes_by_property(
2631 self.REG_NOT_EXISTS, ignore_inexisting_volumes=False
2632 )
2633 for volume_uuid, not_exists in existing_volumes.items():
2634 properties.namespace = self._build_volume_namespace(volume_uuid)
2636 src_uuid = properties.get(self.PROP_UPDATING_UUID_SRC)
2637 if src_uuid:
2638 self._logger(
2639 'Ignoring volume during manager initialization with prop '
2640 ' PROP_UPDATING_UUID_SRC: {} (properties={})'
2641 .format(
2642 volume_uuid,
2643 self._get_filtered_properties(properties)
2644 )
2645 )
2646 continue
2648 # Insert volume in list if the volume exists. Or if the volume
2649 # is being created and a slave wants to use it (repair = False).
2650 #
2651 # If we are on the master and if repair is True and state is
2652 # Creating, it's probably a bug or crash: the creation process has
2653 # been stopped.
2654 if not_exists == self.STATE_EXISTS or (
2655 not repair and not_exists == self.STATE_CREATING
2656 ):
2657 self._volumes.add(volume_uuid)
2658 continue
2660 if not repair:
2661 self._logger(
2662 'Ignoring bad volume during manager initialization: {} '
2663 '(properties={})'.format(
2664 volume_uuid,
2665 self._get_filtered_properties(properties)
2666 )
2667 )
2668 continue
2670 # Remove bad volume.
2671 try:
2672 self._logger(
2673 'Removing bad volume during manager initialization: {} '
2674 '(properties={})'.format(
2675 volume_uuid,
2676 self._get_filtered_properties(properties)
2677 )
2678 )
2679 volume_name = properties.get(self.PROP_VOLUME_NAME)
2681 # Little optimization, don't call `self._destroy_volume`,
2682 # we already have resource name list.
2683 if volume_name in resource_names:
2684 self._destroy_resource(volume_name, force=True)
2686 # Assume this call is atomic.
2687 properties.clear()
2688 except Exception as e:
2689 # Do not raise, we don't want to block user action.
2690 self._logger(
2691 'Cannot clean volume {}: {}'.format(volume_uuid, e)
2692 )
2694 # The volume can't be removed, maybe it's still in use,
2695 # in this case rename it with the "DELETED_" prefix.
2696 # This prefix is mandatory if it exists a snap transaction to
2697 # rollback because the original VDI UUID can try to be renamed
2698 # with the UUID we are trying to delete...
2699 if not volume_uuid.startswith('DELETED_'):
2700 self.update_volume_uuid(
2701 volume_uuid, 'DELETED_' + volume_uuid, force=True
2702 )
2704 for dest_uuid, src_uuid in updating_uuid_volumes.items():
2705 dest_namespace = self._build_volume_namespace(dest_uuid)
2707 properties.namespace = dest_namespace
2708 if int(properties.get(self.PROP_NOT_EXISTS)):
2709 properties.clear()
2710 continue
2712 properties.namespace = self._build_volume_namespace(src_uuid)
2713 properties.clear()
2715 properties.namespace = dest_namespace
2716 properties.pop(self.PROP_UPDATING_UUID_SRC)
2718 if src_uuid in self._volumes:
2719 self._volumes.remove(src_uuid)
2720 self._volumes.add(dest_uuid)
2722 def _get_sr_properties(self):
2723 return self._create_linstor_kv(self._build_sr_namespace())
2725 def _get_volumes_by_property(
2726 self, reg_prop, ignore_inexisting_volumes=True
2727 ):
2728 base_properties = self._get_kv_cache()
2729 base_properties.namespace = self._build_volume_namespace()
2731 volume_properties = {}
2732 for volume_uuid in self._volumes:
2733 volume_properties[volume_uuid] = ''
2735 for key, value in base_properties.items():
2736 res = reg_prop.match(key)
2737 if res:
2738 volume_uuid = res.groups()[0]
2739 if not ignore_inexisting_volumes or \
2740 volume_uuid in self._volumes:
2741 volume_properties[volume_uuid] = value
2743 return volume_properties
2745 def _create_linstor_kv(self, namespace):
2746 return linstor.KV(
2747 self._group_name,
2748 uri=self._linstor.controller_host(),
2749 namespace=namespace
2750 )
2752 def _get_volume_properties(self, volume_uuid):
2753 properties = self._get_kv_cache()
2754 properties.namespace = self._build_volume_namespace(volume_uuid)
2755 return properties
2757 @classmethod
2758 def _list_database_backups(cls, database_backup_dir):
2759 """
2760 List all visible backup files in database_backup_dir.
2761 DATABASE_BACKUP_DIR_MAIN is only available on the Linstor Controller.
2762 DATABASE_BACKUP_DIR_SPARE will list backups previously made when the Host was the Linstor Controller.
2763 This may not be useful information if it is not the Controller anymore.
2764 """
2765 for path in database_backup_dir.glob(DATABASE_BACKUP_NAME_FORMAT.format(
2766 "[0-9]" * 8 + "_" + "[0-9]" * 6, "*") + ".zip"):
2767 try:
2768 yield path, datetime.strptime(path.name.split("-")[1], DATABASE_BACKUP_DATE_FORMAT)
2769 except (ValueError, IndexError):
2770 continue
2772 @classmethod
2773 def _get_sorted_database_backup(cls, database_backup_dir):
2774 """
2775 Return list of backups in database_backup_dir, alongside their creation date.
2776 Sorted by date from the more recent to the older one.
2777 """
2778 return sorted(cls._list_database_backups(database_backup_dir),
2779 reverse=True,
2780 key=lambda p: p[1])
2782 @classmethod
2783 def _get_latest_database_backup(cls):
2784 """
2785 Return the latest backup in DATABASE_BACKUP_DIR_MAIN, and its creation date.
2786 Returns (None, timestamp(0)) when none are found.
2787 None will be found if it is not called on the Linstor Controller.
2788 (cf _list_database_backups)
2789 """
2790 return max(cls._list_database_backups(DATABASE_BACKUP_DIR_MAIN),
2791 default=(None, datetime.fromtimestamp(0)),
2792 key=lambda p: p[1])
2794 @classmethod
2795 def _check_database_backup(cls, database_backup_path):
2796 """
2797 Make some validation of a database backup zip-file.
2798 Check its a valid zipfile, and CRC-test its content.
2799 Check it contains a non-empty linstordb.mv.db file.
2800 Always raises a LinstorDatabaseBackupError if checks failed.
2801 """
2802 try:
2803 with zipfile.ZipFile(database_backup_path, mode="r") as archive:
2804 if archive.testzip() is not None:
2805 raise LinstorDatabaseBackupError("zip archive CRC failed")
2806 linstordb = next((
2807 f
2808 for f in archive.filelist
2809 if f.filename == "linstordb.mv.db"
2810 ), None)
2811 if not linstordb:
2812 raise LinstorDatabaseBackupError("cannot find linstordb.mv.db")
2813 if linstordb.file_size == 0:
2814 raise LinstorDatabaseBackupError("linstordb.mv.db is empty")
2815 except (FileNotFoundError, zipfile.BadZipFile, zipfile.LargeZipFile) as e:
2816 raise LinstorDatabaseBackupError(e) from e
2818 @classmethod
2819 def _build_sr_namespace(cls):
2820 return '/{}/'.format(cls.NAMESPACE_SR)
2822 @classmethod
2823 def _build_volume_namespace(cls, volume_uuid=None):
2824 # Return a path to all volumes if `volume_uuid` is not given.
2825 if volume_uuid is None:
2826 return '/{}/'.format(cls.NAMESPACE_VOLUME)
2827 return '/{}/{}/'.format(cls.NAMESPACE_VOLUME, volume_uuid)
2829 @classmethod
2830 def _get_error_str(cls, result):
2831 return ', '.join([
2832 err.message for err in cls._filter_errors(result)
2833 ])
2835 @classmethod
2836 def _create_linstor_instance(
2837 cls, uri, keep_uri_unmodified=False, attempt_count=30
2838 ):
2839 retry = False
2841 def connect(uri):
2842 if not uri:
2843 uri = get_controller_uri()
2844 if not uri:
2845 raise LinstorVolumeManagerError(
2846 'Unable to find controller uri...'
2847 )
2848 instance = linstor.Linstor(uri, keep_alive=True)
2849 instance.connect()
2850 return instance
2852 try:
2853 return connect(uri)
2854 except (linstor.errors.LinstorNetworkError, LinstorVolumeManagerError):
2855 pass
2857 if not keep_uri_unmodified:
2858 uri = None
2860 return util.retry(
2861 lambda: connect(uri),
2862 maxretry=attempt_count,
2863 period=1,
2864 exceptions=[
2865 linstor.errors.LinstorNetworkError,
2866 LinstorVolumeManagerError
2867 ]
2868 )
2870 @classmethod
2871 def _configure_volume_peer_slots(cls, lin, volume_name):
2872 result = lin.resource_dfn_modify(volume_name, {}, peer_slots=3)
2873 error_str = cls._get_error_str(result)
2874 if error_str:
2875 raise LinstorVolumeManagerError(
2876 'Could not configure volume peer slots of {}: {}'
2877 .format(volume_name, error_str)
2878 )
2880 @classmethod
2881 def _activate_device_path(cls, lin, node_name, volume_name):
2882 result = lin.resource_make_available(node_name, volume_name, diskful=False)
2883 if linstor.Linstor.all_api_responses_no_error(result):
2884 return
2885 errors = linstor.Linstor.filter_api_call_response_errors(result)
2886 if len(errors) == 1 and errors[0].is_error(
2887 linstor.consts.FAIL_EXISTS_RSC
2888 ):
2889 return
2891 raise LinstorVolumeManagerError(
2892 'Unable to activate device path of `{}` on node `{}`: {}'
2893 .format(volume_name, node_name, ', '.join(
2894 [str(x) for x in result]))
2895 )
2897 @classmethod
2898 def _request_database_path(cls, lin, activate=False):
2899 node_name = socket.gethostname()
2901 try:
2902 resource = next(filter(
2903 lambda resource: resource.node_name == node_name and
2904 resource.name == DATABASE_VOLUME_NAME,
2905 lin.resource_list_raise().resources
2906 ), None)
2907 except Exception as e:
2908 raise LinstorVolumeManagerError(
2909 'Unable to fetch database resource: {}'
2910 .format(e)
2911 )
2913 if not resource:
2914 if activate:
2915 cls._activate_device_path(
2916 lin, node_name, DATABASE_VOLUME_NAME
2917 )
2918 return cls._request_database_path(
2919 DATABASE_VOLUME_NAME, DATABASE_VOLUME_NAME
2920 )
2921 raise LinstorVolumeManagerError(
2922 'Empty dev path for `{}`, but definition "seems" to exist'
2923 .format(DATABASE_PATH)
2924 )
2925 # Contains a path of the /dev/drbd<id> form.
2926 return resource.volumes[0].device_path
2928 @classmethod
2929 def _create_database_volume(
2930 cls, lin, group_name, storage_pool_name, node_names, redundancy
2931 ):
2932 try:
2933 dfns = lin.resource_dfn_list_raise().resource_definitions
2934 except Exception as e:
2935 raise LinstorVolumeManagerError(
2936 'Unable to get definitions during database creation: {}'
2937 .format(e)
2938 )
2940 if dfns:
2941 raise LinstorVolumeManagerError(
2942 'Could not create volume `{}` from SR `{}`, '.format(
2943 DATABASE_VOLUME_NAME, group_name
2944 ) + 'LINSTOR volume list must be empty.'
2945 )
2947 # Workaround to use thin lvm. Without this line an error is returned:
2948 # "Not enough available nodes"
2949 # I don't understand why but this command protect against this bug.
2950 try:
2951 pools = lin.storage_pool_list_raise(
2952 filter_by_stor_pools=[storage_pool_name]
2953 )
2954 except Exception as e:
2955 raise LinstorVolumeManagerError(
2956 'Failed to get storage pool list before database creation: {}'
2957 .format(e)
2958 )
2960 # Ensure we have a correct list of storage pools.
2961 assert pools.storage_pools # We must have at least one storage pool!
2962 nodes_with_pool = list(map(lambda pool: pool.node_name, pools.storage_pools))
2963 for node_name in nodes_with_pool:
2964 assert node_name in node_names
2965 util.SMlog('Nodes with storage pool: {}'.format(nodes_with_pool))
2967 # Create the database definition.
2968 size = cls.round_up_volume_size(DATABASE_SIZE)
2969 cls._check_volume_creation_errors(lin.resource_group_spawn(
2970 rsc_grp_name=group_name,
2971 rsc_dfn_name=DATABASE_VOLUME_NAME,
2972 vlm_sizes=['{}B'.format(size)],
2973 definitions_only=True
2974 ), DATABASE_VOLUME_NAME, group_name)
2975 cls._configure_volume_peer_slots(lin, DATABASE_VOLUME_NAME)
2977 # Create real resources on the first nodes.
2978 resources = []
2980 diskful_nodes = []
2981 diskless_nodes = []
2982 for node_name in node_names:
2983 if node_name in nodes_with_pool:
2984 diskful_nodes.append(node_name)
2985 else:
2986 diskless_nodes.append(node_name)
2988 assert diskful_nodes
2989 for node_name in diskful_nodes[:redundancy]:
2990 util.SMlog('Create database diskful on {}'.format(node_name))
2991 resources.append(linstor.ResourceData(
2992 node_name=node_name,
2993 rsc_name=DATABASE_VOLUME_NAME,
2994 storage_pool=storage_pool_name
2995 ))
2996 # Create diskless resources on the remaining set.
2997 for node_name in diskful_nodes[redundancy:] + diskless_nodes:
2998 util.SMlog('Create database diskless on {}'.format(node_name))
2999 resources.append(linstor.ResourceData(
3000 node_name=node_name,
3001 rsc_name=DATABASE_VOLUME_NAME,
3002 diskless=True
3003 ))
3005 result = lin.resource_create(resources)
3006 error_str = cls._get_error_str(result)
3007 if error_str:
3008 raise LinstorVolumeManagerError(
3009 'Could not create database volume from SR `{}`: {}'.format(
3010 group_name, error_str
3011 )
3012 )
3014 # Create database and ensure path exists locally and
3015 # on replicated devices.
3016 current_device_path = cls._request_database_path(lin, activate=True)
3018 # Ensure diskless paths exist on other hosts. Otherwise PBDs can't be
3019 # plugged.
3020 for node_name in node_names:
3021 cls._activate_device_path(lin, node_name, DATABASE_VOLUME_NAME)
3023 # We use realpath here to get the /dev/drbd<id> path instead of
3024 # /dev/drbd/by-res/<resource_name>.
3025 expected_device_path = cls.build_device_path(DATABASE_VOLUME_NAME)
3026 util.wait_for_path(expected_device_path, 5)
3028 device_realpath = os.path.realpath(expected_device_path)
3029 if current_device_path != device_realpath:
3030 raise LinstorVolumeManagerError(
3031 'Invalid path, current={}, expected={} (realpath={})'
3032 .format(
3033 current_device_path,
3034 expected_device_path,
3035 device_realpath
3036 )
3037 )
3039 try:
3040 util.retry(
3041 lambda: util.pread2([DATABASE_MKFS, expected_device_path]),
3042 maxretry=5
3043 )
3044 except Exception as e:
3045 raise LinstorVolumeManagerError(
3046 'Failed to execute {} on database volume: {}'
3047 .format(DATABASE_MKFS, e)
3048 )
3050 return expected_device_path
3052 @classmethod
3053 def _destroy_database_volume(cls, lin, group_name):
3054 error_str = cls._get_error_str(
3055 lin.resource_dfn_delete(DATABASE_VOLUME_NAME)
3056 )
3057 if error_str:
3058 raise LinstorVolumeManagerError(
3059 'Could not destroy resource `{}` from SR `{}`: {}'
3060 .format(DATABASE_VOLUME_NAME, group_name, error_str)
3061 )
3063 @classmethod
3064 def _mount_database_volume(cls, volume_path, mount=True, force=False):
3065 try:
3066 # 1. Create a backup config folder.
3067 database_not_empty = bool(os.listdir(DATABASE_PATH))
3068 backup_path = cls._create_database_backup_path()
3070 # 2. Move the config in the mounted volume.
3071 if database_not_empty:
3072 cls._move_files(DATABASE_PATH, backup_path)
3074 cls._mount_volume(volume_path, DATABASE_PATH, mount)
3076 if database_not_empty:
3077 cls._move_files(backup_path, DATABASE_PATH, force)
3079 # 3. Remove useless backup directory.
3080 try:
3081 os.rmdir(backup_path)
3082 except Exception as e:
3083 raise LinstorVolumeManagerError(
3084 'Failed to remove backup path {} of LINSTOR config: {}'
3085 .format(backup_path, e)
3086 )
3087 except Exception as e:
3088 def force_exec(fn):
3089 try:
3090 fn()
3091 except Exception:
3092 pass
3094 if mount == cls.is_controller():
3095 force_exec(lambda: cls._move_files(
3096 DATABASE_PATH, backup_path
3097 ))
3098 force_exec(lambda: cls._mount_volume(
3099 volume_path, DATABASE_PATH, not mount
3100 ))
3102 if mount != cls.is_controller():
3103 force_exec(lambda: cls._move_files(
3104 backup_path, DATABASE_PATH
3105 ))
3107 force_exec(lambda: os.rmdir(backup_path))
3108 raise e
3110 @classmethod
3111 def _force_destroy_database_volume(cls, lin, group_name):
3112 try:
3113 cls._destroy_database_volume(lin, group_name)
3114 except Exception:
3115 pass
3117 @classmethod
3118 def _destroy_storage_pool(cls, lin, group_name, node_name):
3119 def destroy():
3120 result = lin.storage_pool_delete(node_name, group_name)
3121 errors = cls._filter_errors(result)
3122 if cls._check_errors(errors, [
3123 linstor.consts.FAIL_NOT_FOUND_STOR_POOL,
3124 linstor.consts.FAIL_NOT_FOUND_STOR_POOL_DFN
3125 ]):
3126 return
3128 if errors:
3129 raise LinstorVolumeManagerError(
3130 'Failed to destroy SP `{}` on node `{}`: {}'.format(
3131 group_name,
3132 node_name,
3133 cls._get_error_str(errors)
3134 )
3135 )
3137 # We must retry to avoid errors like:
3138 # "can not be deleted as volumes / snapshot-volumes are still using it"
3139 # after LINSTOR database volume destruction.
3140 return util.retry(destroy, maxretry=10)
3142 @classmethod
3143 def _create_resource_group(
3144 cls,
3145 lin,
3146 group_name,
3147 storage_pool_name,
3148 redundancy,
3149 destroy_old_group
3150 ):
3151 rg_creation_attempt = 0
3152 while True:
3153 result = lin.resource_group_create(
3154 name=group_name,
3155 place_count=redundancy,
3156 storage_pool=storage_pool_name,
3157 diskless_on_remaining=False
3158 )
3159 error_str = cls._get_error_str(result)
3160 if not error_str:
3161 break
3163 errors = cls._filter_errors(result)
3164 if destroy_old_group and cls._check_errors(errors, [
3165 linstor.consts.FAIL_EXISTS_RSC_GRP
3166 ]):
3167 rg_creation_attempt += 1
3168 if rg_creation_attempt < 2:
3169 try:
3170 cls._destroy_resource_group(lin, group_name)
3171 except Exception as e:
3172 error_str = 'Failed to destroy old and empty RG: {}'.format(e)
3173 else:
3174 continue
3176 raise LinstorVolumeManagerError(
3177 'Could not create RG `{}`: {}'.format(
3178 group_name, error_str
3179 )
3180 )
3182 result = lin.volume_group_create(group_name)
3183 error_str = cls._get_error_str(result)
3184 if error_str:
3185 raise LinstorVolumeManagerError(
3186 'Could not create VG `{}`: {}'.format(
3187 group_name, error_str
3188 )
3189 )
3191 @classmethod
3192 def _destroy_resource_group(cls, lin, group_name):
3193 def destroy():
3194 result = lin.resource_group_delete(group_name)
3195 errors = cls._filter_errors(result)
3196 if cls._check_errors(errors, [
3197 linstor.consts.FAIL_NOT_FOUND_RSC_GRP
3198 ]):
3199 return
3201 if errors:
3202 raise LinstorVolumeManagerError(
3203 'Failed to destroy RG `{}`: {}'
3204 .format(group_name, cls._get_error_str(errors))
3205 )
3207 return util.retry(destroy, maxretry=10)
3209 @classmethod
3210 def build_group_name(cls, base_name):
3211 # If thin provisioning is used we have a path like this:
3212 # `VG/LV`. "/" is not accepted by LINSTOR.
3213 return '{}{}'.format(cls.PREFIX_SR, base_name.replace('/', '_'))
3215 # Used to store important data in a HA context,
3216 # i.e. a replication count of 3.
3217 @classmethod
3218 def _build_ha_group_name(cls, base_name):
3219 return '{}{}'.format(cls.PREFIX_HA, base_name.replace('/', '_'))
3221 @classmethod
3222 def _check_volume_creation_errors(cls, result, volume_uuid, group_name):
3223 errors = cls._filter_errors(result)
3224 if cls._check_errors(errors, [
3225 linstor.consts.FAIL_EXISTS_RSC, linstor.consts.FAIL_EXISTS_RSC_DFN
3226 ]):
3227 raise LinstorVolumeManagerError(
3228 'Failed to create volume `{}` from SR `{}`, it already exists'
3229 .format(volume_uuid, group_name),
3230 LinstorVolumeManagerError.ERR_VOLUME_EXISTS
3231 )
3233 if cls._check_errors(errors, [linstor.consts.FAIL_NOT_FOUND_RSC_GRP]):
3234 raise LinstorVolumeManagerError(
3235 'Failed to create volume `{}` from SR `{}`, resource group doesn\'t exist'
3236 .format(volume_uuid, group_name),
3237 LinstorVolumeManagerError.ERR_GROUP_NOT_EXISTS
3238 )
3240 if errors:
3241 raise LinstorVolumeManagerError(
3242 'Failed to create volume `{}` from SR `{}`: {}'.format(
3243 volume_uuid,
3244 group_name,
3245 cls._get_error_str(errors)
3246 )
3247 )
3249 @classmethod
3250 def _move_files(cls, src_dir, dest_dir, force=False):
3251 def listdir(dir):
3252 ignored = ['lost+found']
3253 return [file for file in os.listdir(dir) if file not in ignored]
3255 try:
3256 if not force:
3257 files = listdir(dest_dir)
3258 if files:
3259 raise LinstorVolumeManagerError(
3260 'Cannot move files from {} to {} because destination '
3261 'contains: {}'.format(src_dir, dest_dir, files)
3262 )
3263 except LinstorVolumeManagerError:
3264 raise
3265 except Exception as e:
3266 raise LinstorVolumeManagerError(
3267 'Cannot list dir {}: {}'.format(dest_dir, e)
3268 )
3270 try:
3271 for file in listdir(src_dir):
3272 try:
3273 dest_file = os.path.join(dest_dir, file)
3274 if not force and os.path.exists(dest_file):
3275 raise LinstorVolumeManagerError(
3276 'Cannot move {} because it already exists in the '
3277 'destination'.format(file)
3278 )
3279 shutil.move(os.path.join(src_dir, file), dest_file)
3280 except LinstorVolumeManagerError:
3281 raise
3282 except Exception as e:
3283 raise LinstorVolumeManagerError(
3284 'Cannot move {}: {}'.format(file, e)
3285 )
3286 except Exception as e:
3287 if not force:
3288 try:
3289 cls._move_files(dest_dir, src_dir, force=True)
3290 except Exception:
3291 pass
3293 raise LinstorVolumeManagerError(
3294 'Failed to move files from {} to {}: {}'.format(
3295 src_dir, dest_dir, e
3296 )
3297 )
3299 @staticmethod
3300 def _create_database_backup_path():
3301 path = DATABASE_PATH + '-' + str(uuid.uuid4())
3302 try:
3303 os.mkdir(path)
3304 return path
3305 except Exception as e:
3306 raise LinstorVolumeManagerError(
3307 'Failed to create backup path {} of LINSTOR config: {}'
3308 .format(path, e)
3309 )
3311 @staticmethod
3312 def _get_filtered_properties(properties):
3313 return dict(properties.items())
3315 @staticmethod
3316 def _filter_errors(result):
3317 return [
3318 err for err in result
3319 if hasattr(err, 'is_error') and err.is_error()
3320 ]
3322 @staticmethod
3323 def _check_errors(result, codes):
3324 for err in result:
3325 for code in codes:
3326 if err.is_error(code):
3327 return True
3328 return False
3330 @classmethod
3331 def _controller_is_running(cls):
3332 return cls._service_is_running('linstor-controller')
3334 @classmethod
3335 def _start_controller(cls, start=True):
3336 return cls._start_service('linstor-controller', start)
3338 @staticmethod
3339 def _start_service(name, start=True):
3340 action = 'start' if start else 'stop'
3341 (ret, out, err) = util.doexec([
3342 'systemctl', action, name
3343 ])
3344 if ret != 0:
3345 raise LinstorVolumeManagerError(
3346 'Failed to {} {}: {} {}'
3347 .format(action, name, out, err)
3348 )
3350 @staticmethod
3351 def _service_is_running(name):
3352 (ret, out, err) = util.doexec([
3353 'systemctl', 'is-active', '--quiet', name
3354 ])
3355 return not ret
3357 @staticmethod
3358 def _is_mounted(mountpoint):
3359 (ret, out, err) = util.doexec(['mountpoint', '-q', mountpoint])
3360 return ret == 0
3362 @classmethod
3363 def _mount_volume(cls, volume_path, mountpoint, mount=True):
3364 if mount:
3365 try:
3366 util.pread(['mount', volume_path, mountpoint])
3367 except Exception as e:
3368 raise LinstorVolumeManagerError(
3369 'Failed to mount volume {} on {}: {}'
3370 .format(volume_path, mountpoint, e)
3371 )
3372 else:
3373 try:
3374 if cls._is_mounted(mountpoint):
3375 util.pread(['umount', mountpoint])
3376 except Exception as e:
3377 raise LinstorVolumeManagerError(
3378 'Failed to umount volume {} on {}: {}'
3379 .format(volume_path, mountpoint, e)
3380 )
3383# ==============================================================================
3385# Check if a path is a DRBD resource and log the process name/pid
3386# that opened it.
3387def log_drbd_openers(path):
3388 # Ignore if it's not a symlink to DRBD resource.
3389 if not path.startswith(DRBD_BY_RES_PATH):
3390 return
3392 # Compute resource name.
3393 res_name_end = path.find('/', len(DRBD_BY_RES_PATH))
3394 if res_name_end == -1:
3395 return
3396 res_name = path[len(DRBD_BY_RES_PATH):res_name_end]
3398 volume_end = path.rfind('/')
3399 if volume_end == res_name_end:
3400 return
3401 volume = path[volume_end + 1:]
3403 try:
3404 # Ensure path is a DRBD.
3405 drbd_path = os.path.realpath(path)
3406 stats = os.stat(drbd_path)
3407 if not stat.S_ISBLK(stats.st_mode) or os.major(stats.st_rdev) != 147:
3408 return
3410 # Find where the device is open.
3411 (ret, stdout, stderr) = util.doexec(['drbdadm', 'status', res_name])
3412 if ret != 0:
3413 util.SMlog('Failed to execute `drbdadm status` on `{}`: {}'.format(
3414 res_name, stderr
3415 ))
3416 return
3418 # Is it a local device?
3419 if stdout.startswith('{} role:Primary'.format(res_name)):
3420 util.SMlog(
3421 'DRBD resource `{}` is open on local host: {}'
3422 .format(path, get_local_volume_openers(res_name, volume))
3423 )
3424 return
3426 # Is it a remote device?
3427 util.SMlog(
3428 'DRBD resource `{}` is open on hosts: {}'
3429 .format(path, get_all_volume_openers(res_name, volume))
3430 )
3431 except Exception as e:
3432 util.SMlog(
3433 'Got exception while trying to determine where DRBD resource ' +
3434 '`{}` is open: {}'.format(path, e)
3435 )