Coverage for drivers/linstorvolumemanager.py : 11%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/env python3
2#
3# Copyright (C) 2020 Vates SAS - ronan.abhamon@vates.fr
4#
5# This program is free software: you can redistribute it and/or modify
6# it under the terms of the GNU General Public License as published by
7# the Free Software Foundation, either version 3 of the License, or
8# (at your option) any later version.
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU General Public License for more details.
13#
14# You should have received a copy of the GNU General Public License
15# along with this program. If not, see <https://www.gnu.org/licenses/>.
16#
18from sm_typing import (
19 Any,
20 Dict,
21 List,
22 cast,
23 override,
24)
26import json
27import linstor
28import os.path
29import re
30import shutil
31import socket
32import stat
33import time
34import util
35import uuid
36from datetime import datetime
37from pathlib import Path
38import contextlib
39import zipfile
41# Persistent prefix to add to RAW persistent volumes.
42PERSISTENT_PREFIX = 'xcp-persistent-'
44# Contains the data of the "/var/lib/linstor" directory.
45DATABASE_VOLUME_NAME = PERSISTENT_PREFIX + 'database'
46DATABASE_SIZE = 1 << 30 # 1GB.
47DATABASE_PATH = '/var/lib/linstor'
48DATABASE_MKFS = 'mkfs.ext4'
49DATABASE_BACKUP_DIR_MAIN = Path(DATABASE_PATH)
50DATABASE_BACKUP_DIR_SPARE = Path("/var/lib/linstor.d/db-backups")
51DATABASE_BACKUP_NAME_FORMAT = "linstor_database_backup-{}-{}"
52DATABASE_BACKUP_RETENTION = 10
53DATABASE_BACKUP_DATE_FORMAT = "%Y%m%d_%H%M%S"
54LINSTOR_SATELLITE_PORT = 3366
56REG_DRBDADM_PRIMARY = re.compile("([^\\s]+)\\s+role:Primary")
57REG_DRBDSETUP_IP = re.compile('[^\\s]+\\s+(.*):.*$')
59DRBD_BY_RES_PATH = '/dev/drbd/by-res/'
61PLUGIN = 'linstor-manager'
63LinstorLocalVolumeOpeners = Dict[str, Dict[str, Any]]
64LinstorVolumeOpeners = Dict[str, LinstorLocalVolumeOpeners]
66# ==============================================================================
68def get_local_volume_openers(resource_name, volume) -> LinstorLocalVolumeOpeners:
69 if not resource_name or volume is None:
70 raise Exception('Cannot get DRBD openers without resource name and/or volume.')
72 path = '/sys/kernel/debug/drbd/resources/{}/volumes/{}/openers'.format(
73 resource_name, volume
74 )
76 with open(path, 'r') as openers:
77 # Not a big cost, so read all lines directly.
78 lines = openers.readlines()
80 result = {}
82 opener_re = re.compile('(.*)\\s+([0-9]+)\\s+([0-9]+)')
83 for line in lines:
84 match = opener_re.match(line)
85 assert match
87 groups = match.groups()
88 process_name = groups[0]
89 pid = groups[1]
90 open_duration_ms = groups[2]
92 try:
93 cmdline = util.get_process_cmdline(int(pid))
94 except Exception as e:
95 util.SMlog(f"Failed to get command line of `{pid}`: {e}")
96 cmdline = []
98 result[pid] = {
99 'process-name': process_name,
100 'open-duration': open_duration_ms,
101 'cmdline': cmdline
102 }
104 return cast(LinstorLocalVolumeOpeners, json.dumps(result))
106def get_all_volume_openers(resource_name, volume) -> LinstorVolumeOpeners:
107 PLUGIN_CMD = 'getDrbdOpeners'
109 volume = str(volume)
110 openers = {}
112 with util.ApiSession("SM-get-linstor-volume-openers") as session:
113 hosts = session.xenapi.host.get_all_records()
114 for host_ref, host_record in hosts.items():
115 node_name = host_record['hostname']
116 try:
117 if not session.xenapi.host_metrics.get_record(
118 host_record['metrics']
119 )['live']:
120 # Ensure we call plugin on online hosts only.
121 continue
123 openers[node_name] = json.loads(
124 session.xenapi.host.call_plugin(host_ref, PLUGIN, PLUGIN_CMD, {
125 'resourceName': resource_name,
126 'volume': volume
127 })
128 )
129 except Exception as e:
130 util.SMlog('Failed to get openers of `{}` on `{}`: {}'.format(
131 resource_name, node_name, e
132 ))
134 return openers
137# ==============================================================================
139def round_up(value, divisor):
140 assert divisor
141 divisor = int(divisor)
142 return ((int(value) + divisor - 1) // divisor) * divisor
145def round_down(value, divisor):
146 assert divisor
147 value = int(value)
148 return value - (value % int(divisor))
151# ==============================================================================
153def _get_controller_addresses() -> List[str]:
154 try:
155 (ret, stdout, stderr) = util.doexec([
156 "/usr/sbin/ss", "-tnpH", "state", "established", f"( sport = :{LINSTOR_SATELLITE_PORT} )"
157 ])
158 if ret == 0:
159 return [
160 line.split()[3].rsplit(":", 1)[0]
161 for line in stdout.splitlines()
162 ]
163 util.SMlog(f"Unexpected code {ret}: {stderr}")
164 except Exception as e:
165 util.SMlog(f"Unable to get controller addresses: {e}")
166 return []
168def _get_controller_uri() -> str:
169 # TODO: Check that an IP address from the current pool is returned.
170 addresses = _get_controller_addresses()
171 return "linstor://" + addresses[0] if addresses else ""
173def get_controller_uri():
174 retries = 0
175 while True:
176 uri = _get_controller_uri()
177 if uri:
178 return uri
180 retries += 1
181 if retries >= 30:
182 break
183 time.sleep(1)
186def get_controller_node_name():
187 PLUGIN_CMD = 'hasControllerRunning'
189 (ret, stdout, stderr) = util.doexec([
190 'drbdadm', 'status', DATABASE_VOLUME_NAME
191 ])
193 if ret == 0:
194 if stdout.startswith('{} role:Primary'.format(DATABASE_VOLUME_NAME)):
195 return 'localhost'
197 res = REG_DRBDADM_PRIMARY.search(stdout)
198 if res:
199 return res.groups()[0]
201 session = util.timeout(5, util.ApiSession, "SM-get-linstor-controller-node-name").session
203 for host_ref, host_record in session.xenapi.host.get_all_records().items():
204 node_name = host_record['hostname']
205 try:
206 if not session.xenapi.host_metrics.get_record(
207 host_record['metrics']
208 )['live']:
209 continue
211 if util.strtobool(session.xenapi.host.call_plugin(
212 host_ref, PLUGIN, PLUGIN_CMD, {}
213 )):
214 return node_name
215 except Exception as e:
216 util.SMlog('Failed to call plugin to get controller on `{}`: {}'.format(
217 node_name, e
218 ))
221def demote_drbd_resource(node_name, resource_name):
222 PLUGIN_CMD = 'demoteDrbdResource'
224 session = util.timeout(5, util.ApiSession, "SM-demote-drbd-resource").session
226 for host_ref, host_record in session.xenapi.host.get_all_records().items():
227 if host_record['hostname'] != node_name:
228 continue
230 try:
231 session.xenapi.host.call_plugin(
232 host_ref, PLUGIN, PLUGIN_CMD, {'resource_name': resource_name}
233 )
234 except Exception as e:
235 util.SMlog('Failed to demote resource `{}` on `{}`: {}'.format(
236 resource_name, node_name, e
237 ))
238 raise Exception(
239 'Can\'t demote resource `{}`, unable to find node `{}`'
240 .format(resource_name, node_name)
241 )
243# ==============================================================================
245class LinstorVolumeManagerError(Exception):
246 ERR_GENERIC = 0,
247 ERR_VOLUME_EXISTS = 1,
248 ERR_VOLUME_NOT_EXISTS = 2,
249 ERR_VOLUME_DESTROY = 3,
250 ERR_GROUP_NOT_EXISTS = 4,
251 ERR_VOLUME_IN_USE = 5
253 def __init__(self, message, code=ERR_GENERIC):
254 super(LinstorVolumeManagerError, self).__init__(message)
255 self._code = code
257 @property
258 def code(self):
259 return self._code
261class LinstorDatabaseBackupError(Exception):
262 pass
264# ==============================================================================
266# Note:
267# If a storage pool is not accessible after a network change:
268# linstor node interface modify <NODE> default --ip <IP>
271class LinstorVolumeManager(object):
272 """
273 API to manager LINSTOR volumes in XCP-ng.
274 A volume in this context is a physical part of the storage layer.
275 """
277 __slots__ = (
278 '_linstor', '_uri', '_logger', '_redundancy',
279 '_base_group_name', '_group_name', '_ha_group_name',
280 '_volumes', '_storage_pools', '_storage_pools_time',
281 '_kv_cache', '_resource_cache', '_volume_info_cache',
282 '_kv_cache_dirty', '_resource_cache_dirty', '_volume_info_cache_dirty',
283 '_resources_info_cache',
284 )
286 DEV_ROOT_PATH = DRBD_BY_RES_PATH
288 # Default sector size.
289 BLOCK_SIZE = 512
291 # List of volume properties.
292 PROP_METADATA = 'metadata'
293 PROP_NOT_EXISTS = 'not-exists'
294 PROP_VOLUME_NAME = 'volume-name'
295 PROP_IS_READONLY_TIMESTAMP = 'readonly-timestamp'
297 # A volume can only be locked for a limited duration.
298 # The goal is to give enough time to slaves to execute some actions on
299 # a device before an UUID update or a coalesce for example.
300 # Expiration is expressed in seconds.
301 LOCKED_EXPIRATION_DELAY = 1 * 60
303 # Used when volume uuid is being updated.
304 PROP_UPDATING_UUID_SRC = 'updating-uuid-src'
306 # States of property PROP_NOT_EXISTS.
307 STATE_EXISTS = '0'
308 STATE_NOT_EXISTS = '1'
309 STATE_CREATING = '2'
311 # Property namespaces.
312 NAMESPACE_SR = 'xcp/sr'
313 NAMESPACE_VOLUME = 'xcp/volume'
315 # Regex to match properties.
316 REG_PROP = '^([^/]+)/{}$'
318 REG_METADATA = re.compile(REG_PROP.format(PROP_METADATA))
319 REG_NOT_EXISTS = re.compile(REG_PROP.format(PROP_NOT_EXISTS))
320 REG_VOLUME_NAME = re.compile(REG_PROP.format(PROP_VOLUME_NAME))
321 REG_UPDATING_UUID_SRC = re.compile(REG_PROP.format(PROP_UPDATING_UUID_SRC))
323 # Prefixes of SR/VOLUME in the LINSTOR DB.
324 # A LINSTOR (resource, group, ...) name cannot start with a number.
325 # So we add a prefix behind our SR/VOLUME uuids.
326 PREFIX_SR = 'xcp-sr-'
327 PREFIX_HA = 'xcp-ha-'
328 PREFIX_VOLUME = 'xcp-volume-'
330 # Limit request number when storage pool info is asked, we fetch
331 # the current pool status after N elapsed seconds.
332 STORAGE_POOLS_FETCH_INTERVAL = 15
334 @staticmethod
335 def default_logger(*args):
336 print(args)
338 # --------------------------------------------------------------------------
339 # API.
340 # --------------------------------------------------------------------------
342 class VolumeInfo(object):
343 __slots__ = (
344 'name',
345 'allocated_size', # Allocated size, place count is not used.
346 'virtual_size', # Total virtual available size of this volume
347 # (i.e. the user size at creation).
348 'diskful' # Array of nodes that have a diskful volume.
349 )
351 def __init__(self, name):
352 self.name = name
353 self.allocated_size = 0
354 self.virtual_size = 0
355 self.diskful = []
357 @override
358 def __repr__(self) -> str:
359 return 'VolumeInfo("{}", {}, {}, {})'.format(
360 self.name, self.allocated_size, self.virtual_size,
361 self.diskful
362 )
364 # --------------------------------------------------------------------------
366 def __init__(
367 self, uri, group_name, repair=False, logger=default_logger.__func__,
368 attempt_count=30
369 ):
370 """
371 Create a new LinstorVolumeManager object.
372 :param str uri: URI to communicate with the LINSTOR controller.
373 :param str group_name: The SR group name to use.
374 :param bool repair: If true we try to remove bad volumes due to a crash
375 or unexpected behavior.
376 :param function logger: Function to log messages.
377 :param int attempt_count: Number of attempts to join the controller.
378 """
380 self._uri = uri
381 self._linstor = self._create_linstor_instance(
382 uri, attempt_count=attempt_count
383 )
386 mismatched_nodes = [
387 node for node in self._linstor.node_list().pop().nodes if node.connection_status == "VERSION_MISMATCH"
388 ]
390 if mismatched_nodes:
391 raise LinstorVolumeManagerError(
392 "Some linstor nodes are not using the same version. " +
393 f"Incriminated nodes are: {','.join([node.name for node in mismatched_nodes])}"
394 )
396 self._base_group_name = group_name
398 # Ensure group exists.
399 group_name = self.build_group_name(group_name)
400 groups = self._linstor.resource_group_list_raise([group_name]).resource_groups
401 if not groups:
402 raise LinstorVolumeManagerError(
403 'Unable to find `{}` Linstor SR'.format(group_name)
404 )
406 # Ok. ;)
407 self._logger = logger
408 self._redundancy = groups[0].select_filter.place_count
409 self._group_name = group_name
410 self._ha_group_name = self._build_ha_group_name(self._base_group_name)
411 self._volumes = set()
412 self._storage_pools_time = 0
414 # To increase performance and limit request count to LINSTOR services,
415 # we use caches.
416 self._kv_cache = self._create_kv_cache()
417 self._resource_cache = None
418 self._resource_cache_dirty = True
419 self._volume_info_cache = None
420 self._volume_info_cache_dirty = True
421 self._resources_info_cache = None
422 self._build_volumes(repair=repair)
424 @property
425 def uri(self) -> str:
426 return self._uri
428 @property
429 def native_client(self) -> "linstor.Linstor":
430 return self._linstor
432 @property
433 def group_name(self):
434 """
435 Give the used group name.
436 :return: The group name.
437 :rtype: str
438 """
439 return self._base_group_name
441 @property
442 def redundancy(self):
443 """
444 Give the used redundancy.
445 :return: The redundancy.
446 :rtype: int
447 """
448 return self._redundancy
450 @property
451 def volumes(self):
452 """
453 Give the volumes uuid set.
454 :return: The volumes uuid set.
455 :rtype: set(str)
456 """
457 return self._volumes
459 @property
460 def max_volume_size_allowed(self):
461 """
462 Give the max volume size currently available in B.
463 :return: The current size.
464 :rtype: int
465 """
467 candidates = self._find_best_size_candidates()
468 if not candidates:
469 raise LinstorVolumeManagerError(
470 'Failed to get max volume size allowed'
471 )
473 size = candidates[0].max_volume_size
474 if size < 0:
475 raise LinstorVolumeManagerError(
476 'Invalid max volume size allowed given: {}'.format(size)
477 )
478 return self.round_down_volume_size(size * 1024)
480 @property
481 def physical_size(self):
482 """
483 Give the total physical size of the SR.
484 :return: The physical size.
485 :rtype: int
486 """
487 return self._compute_size('total_capacity')
489 @property
490 def physical_free_size(self):
491 """
492 Give the total free physical size of the SR.
493 :return: The physical free size.
494 :rtype: int
495 """
496 return self._compute_size('free_capacity')
498 @property
499 def allocated_volume_size(self):
500 """
501 Give the allocated size for all volumes. The place count is not
502 used here. When thick lvm is used, the size for one volume should
503 be equal to the virtual volume size. With thin lvm, the size is equal
504 or lower to the volume size.
505 :return: The allocated size of all volumes.
506 :rtype: int
507 """
509 # Paths: /res_name/vol_number/size
510 sizes = {}
512 for resource in self._get_resource_cache().resources:
513 if resource.name not in sizes:
514 current = sizes[resource.name] = {}
515 else:
516 current = sizes[resource.name]
518 for volume in resource.volumes:
519 # We ignore diskless pools of the form "DfltDisklessStorPool".
520 if volume.storage_pool_name != self._group_name:
521 continue
523 allocated_size = max(volume.allocated_size, 0)
524 current_allocated_size = current.get(volume.number) or -1
525 if allocated_size > current_allocated_size:
526 current[volume.number] = allocated_size
528 total_size = 0
529 for volumes in sizes.values():
530 for size in volumes.values():
531 total_size += size
533 return total_size * 1024
535 def get_min_physical_size(self):
536 """
537 Give the minimum physical size of the SR.
538 I.e. the size of the smallest disk + the number of pools.
539 :return: The physical min size.
540 :rtype: tuple(int, int)
541 """
542 size = None
543 pool_count = 0
544 for pool in self._get_storage_pools(force=True):
545 space = pool.free_space
546 if space:
547 pool_count += 1
548 current_size = space.total_capacity
549 if current_size < 0:
550 raise LinstorVolumeManagerError(
551 'Failed to get pool total_capacity attr of `{}`'
552 .format(pool.node_name)
553 )
554 if size is None or current_size < size:
555 size = current_size
556 return (pool_count, (size or 0) * 1024)
558 @property
559 def metadata(self):
560 """
561 Get the metadata of the SR.
562 :return: Dictionary that contains metadata.
563 :rtype: dict(str, dict)
564 """
566 sr_properties = self._get_sr_properties()
567 metadata = sr_properties.get(self.PROP_METADATA)
568 if metadata is not None:
569 metadata = json.loads(metadata)
570 if isinstance(metadata, dict):
571 return metadata
572 raise LinstorVolumeManagerError(
573 'Expected dictionary in SR metadata: {}'.format(
574 self._group_name
575 )
576 )
578 return {}
580 @metadata.setter
581 def metadata(self, metadata):
582 """
583 Set the metadata of the SR.
584 :param dict metadata: Dictionary that contains metadata.
585 """
587 assert isinstance(metadata, dict)
588 sr_properties = self._get_sr_properties()
589 sr_properties[self.PROP_METADATA] = json.dumps(metadata)
591 @property
592 def disconnected_hosts(self):
593 """
594 Get the list of disconnected hosts.
595 :return: Set that contains disconnected hosts.
596 :rtype: set(str)
597 """
599 disconnected_hosts = set()
600 for pool in self._get_storage_pools():
601 for report in pool.reports:
602 if report.ret_code & linstor.consts.WARN_NOT_CONNECTED == \
603 linstor.consts.WARN_NOT_CONNECTED:
604 disconnected_hosts.add(pool.node_name)
605 break
606 return disconnected_hosts
608 def check_volume_exists(self, volume_uuid):
609 """
610 Check if a volume exists in the SR.
611 :return: True if volume exists.
612 :rtype: bool
613 """
614 return volume_uuid in self._volumes
616 def create_volume(
617 self,
618 volume_uuid,
619 size,
620 persistent=True,
621 volume_name=None,
622 high_availability=False
623 ):
624 """
625 Create a new volume on the SR.
626 :param str volume_uuid: The volume uuid to use.
627 :param int size: volume size in B.
628 :param bool persistent: If false the volume will be unavailable
629 on the next constructor call LinstorSR(...).
630 :param str volume_name: If set, this name is used in the LINSTOR
631 database instead of a generated name.
632 :param bool high_availability: If set, the volume is created in
633 the HA group.
634 :return: The current device path of the volume.
635 :rtype: str
636 """
638 self._logger('Creating LINSTOR volume {}...'.format(volume_uuid))
639 if not volume_name:
640 volume_name = self.build_volume_name(util.gen_uuid())
641 volume_properties = self._create_volume_with_properties(
642 volume_uuid,
643 volume_name,
644 size,
645 True, # place_resources
646 high_availability
647 )
649 # Volume created! Now try to find the device path.
650 try:
651 self._logger(
652 'Find device path of LINSTOR volume {}...'.format(volume_uuid)
653 )
654 device_path = self._find_device_path(volume_uuid, volume_name)
655 if persistent:
656 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_EXISTS
657 self._volumes.add(volume_uuid)
658 self._logger(
659 'LINSTOR volume {} created!'.format(volume_uuid)
660 )
661 return device_path
662 except Exception:
663 # There is an issue to find the path.
664 # At this point the volume has just been created, so force flag can be used.
665 self._destroy_volume(volume_uuid, force=True)
666 raise
668 def mark_volume_as_persistent(self, volume_uuid):
669 """
670 Mark volume as persistent if created with persistent=False.
671 :param str volume_uuid: The volume uuid to mark.
672 """
674 self._ensure_volume_exists(volume_uuid)
676 # Mark volume as persistent.
677 volume_properties = self._get_volume_properties(volume_uuid)
678 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_EXISTS
680 def destroy_volume(self, volume_uuid):
681 """
682 Destroy a volume.
683 :param str volume_uuid: The volume uuid to destroy.
684 """
686 self._ensure_volume_exists(volume_uuid)
687 self.ensure_volume_is_not_locked(volume_uuid)
689 is_volume_in_use = any(node["in-use"] for node in self.get_resource_info(volume_uuid)["nodes"].values())
690 if is_volume_in_use:
691 raise LinstorVolumeManagerError(
692 f"Could not destroy volume `{volume_uuid}` as it is currently in use",
693 LinstorVolumeManagerError.ERR_VOLUME_IN_USE
694 )
696 # Mark volume as destroyed.
697 volume_properties = self._get_volume_properties(volume_uuid)
698 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_NOT_EXISTS
700 try:
701 self._volumes.remove(volume_uuid)
702 self._destroy_volume(volume_uuid)
703 except Exception as e:
704 raise LinstorVolumeManagerError(
705 str(e),
706 LinstorVolumeManagerError.ERR_VOLUME_DESTROY
707 )
709 def lock_volume(self, volume_uuid, locked=True):
710 """
711 Prevent modifications of the volume properties during
712 "self.LOCKED_EXPIRATION_DELAY" seconds. The SR must be locked
713 when used. This method is useful to attach/detach correctly a volume on
714 a slave. Without it the GC can rename a volume, in this case the old
715 volume path can be used by a slave...
716 :param str volume_uuid: The volume uuid to protect/unprotect.
717 :param bool locked: Lock/unlock the volume.
718 """
720 self._ensure_volume_exists(volume_uuid)
722 self._logger(
723 '{} volume {} as locked'.format(
724 'Mark' if locked else 'Unmark',
725 volume_uuid
726 )
727 )
729 volume_properties = self._get_volume_properties(volume_uuid)
730 if locked:
731 volume_properties[
732 self.PROP_IS_READONLY_TIMESTAMP
733 ] = str(time.time())
734 elif self.PROP_IS_READONLY_TIMESTAMP in volume_properties:
735 volume_properties.pop(self.PROP_IS_READONLY_TIMESTAMP)
737 def ensure_volume_is_not_locked(self, volume_uuid, timeout=None):
738 """
739 Ensure a volume is not locked. Wait if necessary.
740 :param str volume_uuid: The volume uuid to check.
741 :param int timeout: If the volume is always locked after the expiration
742 of the timeout, an exception is thrown.
743 """
744 return self.ensure_volume_list_is_not_locked([volume_uuid], timeout)
746 def ensure_volume_list_is_not_locked(self, volume_uuids, timeout=None):
747 checked = set()
748 for volume_uuid in volume_uuids:
749 if volume_uuid in self._volumes:
750 checked.add(volume_uuid)
752 if not checked:
753 return
755 waiting = False
757 volume_properties = self._get_kv_cache()
759 start = time.time()
760 while True:
761 # Can't delete in for loop, use a copy of the list.
762 remaining = checked.copy()
763 for volume_uuid in checked:
764 volume_properties.namespace = \
765 self._build_volume_namespace(volume_uuid)
766 timestamp = volume_properties.get(
767 self.PROP_IS_READONLY_TIMESTAMP
768 )
769 if timestamp is None:
770 remaining.remove(volume_uuid)
771 continue
773 now = time.time()
774 if now - float(timestamp) > self.LOCKED_EXPIRATION_DELAY:
775 self._logger(
776 'Remove readonly timestamp on {}'.format(volume_uuid)
777 )
778 volume_properties.pop(self.PROP_IS_READONLY_TIMESTAMP)
779 remaining.remove(volume_uuid)
780 continue
782 if not waiting:
783 self._logger(
784 'Volume {} is locked, waiting...'.format(volume_uuid)
785 )
786 waiting = True
787 break
789 if not remaining:
790 break
791 checked = remaining
793 if timeout is not None and now - start > timeout:
794 raise LinstorVolumeManagerError(
795 'volume `{}` is locked and timeout has been reached'
796 .format(volume_uuid),
797 LinstorVolumeManagerError.ERR_VOLUME_NOT_EXISTS
798 )
800 # We must wait to use the volume. After that we can modify it
801 # ONLY if the SR is locked to avoid bad reads on the slaves.
802 time.sleep(1)
803 volume_properties = self._create_kv_cache()
805 if waiting:
806 self._logger('No volume locked now!')
808 def remove_volume_if_diskless(self, volume_uuid):
809 """
810 Remove disless path from local node.
811 :param str volume_uuid: The volume uuid to remove.
812 """
814 self._ensure_volume_exists(volume_uuid)
816 volume_properties = self._get_volume_properties(volume_uuid)
817 volume_name = volume_properties.get(self.PROP_VOLUME_NAME)
819 node_name = socket.gethostname()
821 for resource in self._get_resource_cache().resources:
822 if resource.name == volume_name and resource.node_name == node_name:
823 if linstor.consts.FLAG_TIE_BREAKER in resource.flags:
824 return
825 break
827 result = self._linstor.resource_delete_if_diskless(
828 node_name=node_name, rsc_name=volume_name
829 )
830 if not linstor.Linstor.all_api_responses_no_error(result):
831 raise LinstorVolumeManagerError(
832 'Unable to delete diskless path of `{}` on node `{}`: {}'
833 .format(volume_name, node_name, ', '.join(
834 [str(x) for x in result]))
835 )
837 def introduce_volume(self, volume_uuid):
838 pass # TODO: Implement me.
840 def resize_volume(self, volume_uuid, new_size):
841 """
842 Resize a volume.
843 :param str volume_uuid: The volume uuid to resize.
844 :param int new_size: New size in B.
845 """
847 volume_name = self.get_volume_name(volume_uuid)
848 self.ensure_volume_is_not_locked(volume_uuid)
849 new_size = self.round_up_volume_size(new_size) // 1024
851 # We can't resize anything until DRBD is up to date.
852 # We wait here for 5min max and raise an easy to understand error for the user.
853 # 5min is an arbitrary time, it's impossible to get a fit all situation value
854 # and it's currently impossible to know how much time we have to wait
855 # This is mostly an issue for thick provisioning, thin isn't affected.
856 start_time = time.monotonic()
857 try:
858 self._linstor.resource_dfn_wait_synced(volume_name, wait_interval=1.0, timeout=60*5)
859 except linstor.LinstorTimeoutError:
860 raise LinstorVolumeManagerError(
861 f"Volume resizing of `{volume_uuid}` from SR `{self._group_name}` is incomplete: timeout reached but it continues in background."
862 )
863 util.SMlog(f"DRBD is up to date, syncing took {time.monotonic() - start_time}s")
865 result = self._linstor.volume_dfn_modify(
866 rsc_name=volume_name,
867 volume_nr=0,
868 size=new_size
869 )
871 self._mark_resource_cache_as_dirty()
873 error_str = self._get_error_str(result)
874 if error_str:
875 raise LinstorVolumeManagerError(
876 f"Could not resize volume `{volume_uuid}` from SR `{self._group_name}`: {error_str}"
877 )
879 def get_volume_name(self, volume_uuid):
880 """
881 Get the name of a particular volume.
882 :param str volume_uuid: The volume uuid of the name to get.
883 :return: The volume name.
884 :rtype: str
885 """
887 self._ensure_volume_exists(volume_uuid)
888 volume_properties = self._get_volume_properties(volume_uuid)
889 volume_name = volume_properties.get(self.PROP_VOLUME_NAME)
890 if volume_name:
891 return volume_name
892 raise LinstorVolumeManagerError(
893 'Failed to get volume name of {}'.format(volume_uuid)
894 )
896 def get_volume_size(self, volume_uuid):
897 """
898 Get the size of a particular volume.
899 :param str volume_uuid: The volume uuid of the size to get.
900 :return: The volume size.
901 :rtype: int
902 """
904 volume_name = self.get_volume_name(volume_uuid)
905 dfns = self._linstor.resource_dfn_list_raise(
906 query_volume_definitions=True,
907 filter_by_resource_definitions=[volume_name]
908 ).resource_definitions
910 size = dfns[0].volume_definitions[0].size
911 if size < 0:
912 raise LinstorVolumeManagerError(
913 'Failed to get volume size of: {}'.format(volume_uuid)
914 )
915 return size * 1024
917 def set_auto_promote_timeout(self, volume_uuid, timeout):
918 """
919 Define the blocking time of open calls when a DRBD
920 is already open on another host.
921 :param str volume_uuid: The volume uuid to modify.
922 """
924 volume_name = self.get_volume_name(volume_uuid)
925 result = self._linstor.resource_dfn_modify(volume_name, {
926 'DrbdOptions/Resource/auto-promote-timeout': timeout
927 })
928 error_str = self._get_error_str(result)
929 if error_str:
930 raise LinstorVolumeManagerError(
931 'Could not change the auto promote timeout of `{}`: {}'
932 .format(volume_uuid, error_str)
933 )
935 def set_drbd_ha_properties(self, volume_name, enabled=True):
936 """
937 Set or not HA DRBD properties required by drbd-reactor and
938 by specific volumes.
939 :param str volume_name: The volume to modify.
940 :param bool enabled: Enable or disable HA properties.
941 """
943 properties = {
944 'DrbdOptions/auto-quorum': 'disabled',
945 'DrbdOptions/Resource/auto-promote': 'no',
946 'DrbdOptions/Resource/on-no-data-accessible': 'io-error',
947 'DrbdOptions/Resource/on-no-quorum': 'io-error',
948 'DrbdOptions/Resource/on-suspended-primary-outdated': 'force-secondary',
949 'DrbdOptions/Resource/quorum': 'majority'
950 }
951 if enabled:
952 result = self._linstor.resource_dfn_modify(volume_name, properties)
953 else:
954 result = self._linstor.resource_dfn_modify(volume_name, {}, delete_props=list(properties.keys()))
956 error_str = self._get_error_str(result)
957 if error_str:
958 raise LinstorVolumeManagerError(
959 'Could not modify HA DRBD properties on volume `{}`: {}'
960 .format(volume_name, error_str)
961 )
963 def get_volume_info(self, volume_uuid):
964 """
965 Get the volume info of a particular volume.
966 :param str volume_uuid: The volume uuid of the volume info to get.
967 :return: The volume info.
968 :rtype: VolumeInfo
969 """
971 volume_name = self.get_volume_name(volume_uuid)
972 try:
973 return self._get_volumes_info()[volume_name]
974 except KeyError:
975 raise LinstorVolumeManagerError(
976 f"Could not find info about volume `{volume_uuid}`, the resource definition has likely been deleted",
977 LinstorVolumeManagerError.ERR_VOLUME_NOT_EXISTS
978 )
980 def get_device_path(self, volume_uuid):
981 """
982 Get the dev path of a volume, create a diskless if necessary.
983 :param str volume_uuid: The volume uuid to get the dev path.
984 :return: The current device path of the volume.
985 :rtype: str
986 """
988 volume_name = self.get_volume_name(volume_uuid)
989 return self._find_device_path(volume_uuid, volume_name)
991 def get_volume_uuid_from_device_path(self, device_path):
992 """
993 Get the volume uuid of a device_path.
994 :param str device_path: The dev path to find the volume uuid.
995 :return: The volume uuid of the local device path.
996 :rtype: str
997 """
999 expected_volume_name = \
1000 self.get_volume_name_from_device_path(device_path)
1002 volume_names = self.get_volumes_with_name()
1003 for volume_uuid, volume_name in volume_names.items():
1004 if volume_name == expected_volume_name:
1005 return volume_uuid
1007 raise LinstorVolumeManagerError(
1008 'Unable to find volume uuid from dev path `{}`'.format(device_path)
1009 )
1011 def get_volume_name_from_device_path(self, device_path):
1012 """
1013 Get the volume name of a device_path.
1014 :param str device_path: The dev path to find the volume name.
1015 :return: The volume name of the device path.
1016 :rtype: str
1017 """
1019 # Assume that we have a path like this:
1020 # - "/dev/drbd/by-res/xcp-volume-<UUID>/0"
1021 # - "../xcp-volume-<UUID>/0"
1022 if device_path.startswith(DRBD_BY_RES_PATH):
1023 prefix_len = len(DRBD_BY_RES_PATH)
1024 elif device_path.startswith('../'):
1025 prefix_len = 3
1026 else:
1027 raise LinstorVolumeManagerError('Unexpected device path: `{}`'.format(device_path))
1029 res_name_end = device_path.find('/', prefix_len)
1030 assert res_name_end != -1
1031 return device_path[prefix_len:res_name_end]
1033 def update_volume_uuid(self, volume_uuid, new_volume_uuid, force=False):
1034 """
1035 Change the uuid of a volume.
1036 :param str volume_uuid: The volume to modify.
1037 :param str new_volume_uuid: The new volume uuid to use.
1038 :param bool force: If true we doesn't check if volume_uuid is in the
1039 volume list. I.e. the volume can be marked as deleted but the volume
1040 can still be in the LINSTOR KV store if the deletion has failed.
1041 In specific cases like "undo" after a failed clone we must rename a bad
1042 deleted VDI.
1043 """
1045 self._logger(
1046 'Trying to update volume UUID {} to {}...'
1047 .format(volume_uuid, new_volume_uuid)
1048 )
1049 assert volume_uuid != new_volume_uuid, 'can\'t update volume UUID, same value'
1051 if not force:
1052 self._ensure_volume_exists(volume_uuid)
1053 self.ensure_volume_is_not_locked(volume_uuid)
1055 if new_volume_uuid in self._volumes:
1056 raise LinstorVolumeManagerError(
1057 'Volume `{}` already exists'.format(new_volume_uuid),
1058 LinstorVolumeManagerError.ERR_VOLUME_EXISTS
1059 )
1061 volume_properties = self._get_volume_properties(volume_uuid)
1062 if volume_properties.get(self.PROP_UPDATING_UUID_SRC):
1063 raise LinstorVolumeManagerError(
1064 'Cannot update volume uuid {}: invalid state'
1065 .format(volume_uuid)
1066 )
1068 # 1. Copy in temp variables metadata and volume_name.
1069 metadata = volume_properties.get(self.PROP_METADATA)
1070 volume_name = volume_properties.get(self.PROP_VOLUME_NAME)
1072 # 2. Switch to new volume namespace.
1073 volume_properties.namespace = self._build_volume_namespace(
1074 new_volume_uuid
1075 )
1077 if list(volume_properties.items()):
1078 raise LinstorVolumeManagerError(
1079 'Cannot update volume uuid {} to {}: '
1080 .format(volume_uuid, new_volume_uuid) +
1081 'this last one is not empty'
1082 )
1084 try:
1085 # 3. Mark new volume properties with PROP_UPDATING_UUID_SRC.
1086 # If we crash after that, the new properties can be removed
1087 # properly.
1088 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_NOT_EXISTS
1089 volume_properties[self.PROP_UPDATING_UUID_SRC] = volume_uuid
1091 # 4. Copy the properties.
1092 # Note: On new volumes, during clone for example, the metadata
1093 # may be missing. So we must test it to avoid this error:
1094 # "None has to be a str/unicode, but is <type 'NoneType'>"
1095 if metadata:
1096 volume_properties[self.PROP_METADATA] = metadata
1097 volume_properties[self.PROP_VOLUME_NAME] = volume_name
1099 # 5. Ok!
1100 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_EXISTS
1101 except Exception as err:
1102 try:
1103 # Clear the new volume properties in case of failure.
1104 assert volume_properties.namespace == \
1105 self._build_volume_namespace(new_volume_uuid)
1106 volume_properties.clear()
1107 except Exception as e:
1108 self._logger(
1109 'Failed to clear new volume properties: {} (ignoring...)'
1110 .format(e)
1111 )
1112 raise LinstorVolumeManagerError(
1113 'Failed to copy volume properties: {}'.format(err)
1114 )
1116 try:
1117 # 6. After this point, it's ok we can remove the
1118 # PROP_UPDATING_UUID_SRC property and clear the src properties
1119 # without problems.
1121 # 7. Switch to old volume namespace.
1122 volume_properties.namespace = self._build_volume_namespace(
1123 volume_uuid
1124 )
1125 volume_properties.clear()
1127 # 8. Switch a last time to new volume namespace.
1128 volume_properties.namespace = self._build_volume_namespace(
1129 new_volume_uuid
1130 )
1131 volume_properties.pop(self.PROP_UPDATING_UUID_SRC)
1132 except Exception as e:
1133 raise LinstorVolumeManagerError(
1134 'Failed to clear volume properties '
1135 'after volume uuid update: {}'.format(e)
1136 )
1138 try:
1139 self._volumes.remove(volume_uuid)
1140 except KeyError:
1141 # Can be missing if we are building the volume set attr AND
1142 # we are processing a deleted resource.
1143 assert force
1145 self._volumes.add(new_volume_uuid)
1147 self._logger(
1148 'UUID update succeeded of {} to {}! (properties={})'
1149 .format(
1150 volume_uuid, new_volume_uuid,
1151 self._get_filtered_properties(volume_properties)
1152 )
1153 )
1155 def update_volume_name(self, volume_uuid, volume_name):
1156 """
1157 Change the volume name of a volume.
1158 :param str volume_uuid: The volume to modify.
1159 :param str volume_name: The volume_name to use.
1160 """
1162 self._ensure_volume_exists(volume_uuid)
1163 self.ensure_volume_is_not_locked(volume_uuid)
1164 if not volume_name.startswith(self.PREFIX_VOLUME):
1165 raise LinstorVolumeManagerError(
1166 'Volume name `{}` must be start with `{}`'
1167 .format(volume_name, self.PREFIX_VOLUME)
1168 )
1170 if volume_name not in self._fetch_resource_names():
1171 raise LinstorVolumeManagerError(
1172 'Volume `{}` doesn\'t exist'.format(volume_name)
1173 )
1175 volume_properties = self._get_volume_properties(volume_uuid)
1176 volume_properties[self.PROP_VOLUME_NAME] = volume_name
1178 def get_usage_states(self, volume_uuid):
1179 """
1180 Check if a volume is currently used.
1181 :param str volume_uuid: The volume uuid to check.
1182 :return: A dictionary that contains states.
1183 :rtype: dict(str, bool or None)
1184 """
1186 states = {}
1188 volume_name = self.get_volume_name(volume_uuid)
1189 for resource_state in self._linstor.resource_list_raise(
1190 filter_by_resources=[volume_name]
1191 ).resource_states:
1192 states[resource_state.node_name] = resource_state.in_use
1194 return states
1196 def get_volume_openers(self, volume_uuid) -> LinstorVolumeOpeners:
1197 """
1198 Get openers of a volume.
1199 :param str volume_uuid: The volume uuid to monitor.
1200 :return: A dictionary that contains openers.
1201 :rtype: dict(str, obj)
1202 """
1203 return get_all_volume_openers(self.get_volume_name(volume_uuid), '0')
1205 def get_volumes_with_name(self):
1206 """
1207 Give a volume dictionary that contains names actually owned.
1208 :return: A volume/name dict.
1209 :rtype: dict(str, str)
1210 """
1211 return self._get_volumes_by_property(self.REG_VOLUME_NAME)
1213 def get_volumes_with_info(self):
1214 """
1215 Give a volume dictionary that contains VolumeInfos.
1216 :return: A volume/VolumeInfo dict.
1217 :rtype: dict(str, VolumeInfo)
1218 """
1220 volumes = {}
1222 volume_names = self.get_volumes_with_name()
1223 all_volume_info = self._get_volumes_info(volume_names)
1224 for volume_uuid, volume_name in volume_names.items():
1225 if volume_name:
1226 volume_info = all_volume_info.get(volume_name)
1227 if volume_info:
1228 volumes[volume_uuid] = volume_info
1229 continue
1231 # Well I suppose if this volume is not available,
1232 # LINSTOR has been used directly without using this API.
1233 volumes[volume_uuid] = self.VolumeInfo('')
1235 return volumes
1237 def get_volumes_with_metadata(self):
1238 """
1239 Give a volume dictionary that contains metadata.
1240 :return: A volume/metadata dict.
1241 :rtype: dict(str, dict)
1242 """
1244 volumes = {}
1246 metadata = self._get_volumes_by_property(self.REG_METADATA)
1247 for volume_uuid, volume_metadata in metadata.items():
1248 if volume_metadata:
1249 volume_metadata = json.loads(volume_metadata)
1250 if isinstance(volume_metadata, dict):
1251 volumes[volume_uuid] = volume_metadata
1252 continue
1253 raise LinstorVolumeManagerError(
1254 'Expected dictionary in volume metadata: {}'
1255 .format(volume_uuid)
1256 )
1258 volumes[volume_uuid] = {}
1260 return volumes
1262 def get_volume_metadata(self, volume_uuid):
1263 """
1264 Get the metadata of a volume.
1265 :return: Dictionary that contains metadata.
1266 :rtype: dict
1267 """
1269 self._ensure_volume_exists(volume_uuid)
1270 volume_properties = self._get_volume_properties(volume_uuid)
1271 metadata = volume_properties.get(self.PROP_METADATA)
1272 if metadata:
1273 metadata = json.loads(metadata)
1274 if isinstance(metadata, dict):
1275 return metadata
1276 raise LinstorVolumeManagerError(
1277 'Expected dictionary in volume metadata: {}'
1278 .format(volume_uuid)
1279 )
1280 return {}
1282 def set_volume_metadata(self, volume_uuid, metadata):
1283 """
1284 Set the metadata of a volume.
1285 :param dict metadata: Dictionary that contains metadata.
1286 """
1288 self._ensure_volume_exists(volume_uuid)
1289 self.ensure_volume_is_not_locked(volume_uuid)
1291 assert isinstance(metadata, dict)
1292 volume_properties = self._get_volume_properties(volume_uuid)
1293 volume_properties[self.PROP_METADATA] = json.dumps(metadata)
1295 def update_volume_metadata(self, volume_uuid, metadata):
1296 """
1297 Update the metadata of a volume. It modify only the given keys.
1298 It doesn't remove unreferenced key instead of set_volume_metadata.
1299 :param dict metadata: Dictionary that contains metadata.
1300 """
1302 self._ensure_volume_exists(volume_uuid)
1303 self.ensure_volume_is_not_locked(volume_uuid)
1305 assert isinstance(metadata, dict)
1306 volume_properties = self._get_volume_properties(volume_uuid)
1308 current_metadata = json.loads(
1309 volume_properties.get(self.PROP_METADATA, '{}')
1310 )
1311 if not isinstance(metadata, dict):
1312 raise LinstorVolumeManagerError(
1313 'Expected dictionary in volume metadata: {}'
1314 .format(volume_uuid)
1315 )
1317 for key, value in metadata.items():
1318 current_metadata[key] = value
1319 volume_properties[self.PROP_METADATA] = json.dumps(current_metadata)
1321 def shallow_clone_volume(self, volume_uuid, clone_uuid, persistent=True):
1322 """
1323 Clone a volume. Do not copy the data, this method creates a new volume
1324 with the same size.
1325 :param str volume_uuid: The volume to clone.
1326 :param str clone_uuid: The cloned volume.
1327 :param bool persistent: If false the volume will be unavailable
1328 on the next constructor call LinstorSR(...).
1329 :return: The current device path of the cloned volume.
1330 :rtype: str
1331 """
1333 volume_name = self.get_volume_name(volume_uuid)
1334 self.ensure_volume_is_not_locked(volume_uuid)
1336 # 1. Find ideal nodes + size to use.
1337 ideal_node_names, size = self._get_volume_node_names_and_size(
1338 volume_name
1339 )
1340 if size <= 0:
1341 raise LinstorVolumeManagerError(
1342 'Invalid size of {} for volume `{}`'.format(size, volume_name)
1343 )
1345 # 2. Create clone!
1346 return self.create_volume(clone_uuid, size, persistent)
1348 def remove_resourceless_volumes(self):
1349 """
1350 Remove all volumes without valid or non-empty name
1351 (i.e. without LINSTOR resource). It's different than
1352 LinstorVolumeManager constructor that takes a `repair` param that
1353 removes volumes with `PROP_NOT_EXISTS` to 1.
1354 """
1356 resource_names = self._fetch_resource_names()
1357 for volume_uuid, volume_name in self.get_volumes_with_name().items():
1358 if not volume_name or volume_name not in resource_names:
1359 # Don't force, we can be sure of what's happening.
1360 self.destroy_volume(volume_uuid)
1362 def destroy(self):
1363 """
1364 Destroy this SR. Object should not be used after that.
1365 :param bool force: Try to destroy volumes before if true.
1366 """
1368 # 1. Ensure volume list is empty. No cost.
1369 if self._volumes:
1370 raise LinstorVolumeManagerError(
1371 'Cannot destroy LINSTOR volume manager: '
1372 'It exists remaining volumes'
1373 )
1375 # 2. Fetch ALL resource names.
1376 # This list may therefore contain volumes created outside
1377 # the scope of the driver.
1378 resource_names = self._fetch_resource_names(ignore_deleted=False)
1379 try:
1380 resource_names.remove(DATABASE_VOLUME_NAME)
1381 except KeyError:
1382 # Really strange to reach that point.
1383 # Normally we always have the database volume in the list.
1384 pass
1386 # 3. Ensure the resource name list is entirely empty...
1387 if resource_names:
1388 raise LinstorVolumeManagerError(
1389 'Cannot destroy LINSTOR volume manager: '
1390 'It exists remaining volumes (created externally or being deleted)'
1391 )
1393 # 4. Destroying...
1394 controller_is_running = self._controller_is_running()
1395 uri = 'linstor://localhost'
1396 try:
1397 if controller_is_running:
1398 self._start_controller(start=False)
1400 # 4.1. Umount LINSTOR database.
1401 self._mount_database_volume(
1402 self.build_device_path(DATABASE_VOLUME_NAME),
1403 mount=False,
1404 force=True
1405 )
1407 # 4.2. Refresh instance.
1408 self._start_controller(start=True)
1409 self._linstor = self._create_linstor_instance(
1410 uri, keep_uri_unmodified=True
1411 )
1413 # 4.3. Destroy database volume.
1414 self._destroy_resource(DATABASE_VOLUME_NAME)
1416 # 4.4. Refresh linstor connection.
1417 # Without we get this error:
1418 # "Cannot delete resource group 'xcp-sr-linstor_group_thin_device' because it has existing resource definitions.."
1419 # Because the deletion of the databse was not seen by Linstor for some reason.
1420 # It seems a simple refresh of the Linstor connection make it aware of the deletion.
1421 self._linstor.disconnect()
1422 self._linstor.connect()
1424 # 4.5. Destroy remaining drbd nodes on hosts.
1425 # We check if there is a DRBD node on hosts that could mean blocking when destroying resource groups.
1426 # It needs to be done locally by each host so we go through the linstor-manager plugin.
1427 # If we don't do this sometimes, the destroy will fail when trying to destroy the resource groups with:
1428 # "linstor-manager:destroy error: Failed to destroy SP `xcp-sr-linstor_group_thin_device` on node `r620-s2`: The specified storage pool 'xcp-sr-linstor_group_thin_device' on node 'r620-s2' can not be deleted as volumes / snapshot-volumes are still using it."
1429 session = util.timeout(5, util.ApiSession, "SM-linstor-destroy").session
1430 for host_ref in session.xenapi.host.get_all():
1431 try:
1432 response = session.xenapi.host.call_plugin(
1433 host_ref, 'linstor-manager', 'destroyDrbdVolumes', {'volume_group': self._group_name}
1434 )
1435 except Exception as e:
1436 util.SMlog('Calling destroyDrbdVolumes on host {} failed with error {}'.format(host_ref, e))
1438 # 4.6. Destroy group and storage pools.
1439 self._destroy_resource_group(self._linstor, self._group_name)
1440 self._destroy_resource_group(self._linstor, self._ha_group_name)
1441 for pool in self._get_storage_pools(force=True):
1442 self._destroy_storage_pool(
1443 self._linstor, pool.name, pool.node_name
1444 )
1445 except Exception as e:
1446 self._start_controller(start=controller_is_running)
1447 raise e
1449 try:
1450 self._start_controller(start=False)
1451 for file in os.listdir(DATABASE_PATH):
1452 if file != 'lost+found':
1453 os.remove(DATABASE_PATH + '/' + file)
1454 except Exception as e:
1455 util.SMlog(
1456 'Ignoring failure after LINSTOR SR destruction: {}'
1457 .format(e)
1458 )
1460 def find_up_to_date_diskful_nodes(self, volume_uuid):
1461 """
1462 Find all nodes that contain a specific volume using diskful disks.
1463 The disk must be up to data to be used.
1464 :param str volume_uuid: The volume to use.
1465 :return: The available nodes.
1466 :rtype: tuple(set(str), str)
1467 """
1469 volume_name = self.get_volume_name(volume_uuid)
1471 in_use_by = None
1472 node_names = set()
1474 resource_states = filter(
1475 lambda resource_state: resource_state.name == volume_name,
1476 self._get_resource_cache().resource_states
1477 )
1479 for resource_state in resource_states:
1480 volume_state = resource_state.volume_states[0]
1481 if volume_state.disk_state == 'UpToDate':
1482 node_names.add(resource_state.node_name)
1483 if resource_state.in_use:
1484 in_use_by = resource_state.node_name
1486 return (node_names, in_use_by)
1488 def invalidate_resource_cache(self):
1489 """
1490 If resources are impacted by external commands like vhdutil,
1491 it's necessary to call this function to invalidate current resource
1492 cache.
1493 """
1494 self._mark_resource_cache_as_dirty()
1496 def has_node(self, node_name):
1497 """
1498 Check if a node exists in the LINSTOR database.
1499 :rtype: bool
1500 """
1501 result = self._linstor.node_list()
1502 error_str = self._get_error_str(result)
1503 if error_str:
1504 raise LinstorVolumeManagerError(
1505 'Failed to list nodes using `{}`: {}'
1506 .format(node_name, error_str)
1507 )
1508 return bool(result[0].node(node_name))
1510 def create_node(self, node_name, ip):
1511 """
1512 Create a new node in the LINSTOR database.
1513 :param str node_name: Node name to use.
1514 :param str ip: Host IP to communicate.
1515 """
1516 result = self._linstor.node_create(
1517 node_name,
1518 linstor.consts.VAL_NODE_TYPE_CMBD,
1519 ip
1520 )
1521 errors = self._filter_errors(result)
1522 if errors:
1523 error_str = self._get_error_str(errors)
1524 raise LinstorVolumeManagerError(
1525 'Failed to create node `{}`: {}'.format(node_name, error_str)
1526 )
1528 def destroy_node(self, node_name):
1529 """
1530 Destroy a node in the LINSTOR database.
1531 :param str node_name: Node name to remove.
1532 """
1533 result = self._linstor.node_delete(node_name)
1534 errors = self._filter_errors(result)
1535 if errors:
1536 error_str = self._get_error_str(errors)
1537 raise LinstorVolumeManagerError(
1538 'Failed to destroy node `{}`: {}'.format(node_name, error_str)
1539 )
1541 def create_node_interface(self, node_name, name, ip):
1542 """
1543 Create a new node interface in the LINSTOR database.
1544 :param str node_name: Node name of the interface to use.
1545 :param str name: Interface to create.
1546 :param str ip: IP of the interface.
1547 """
1548 result = self._linstor.netinterface_create(node_name, name, ip)
1549 errors = self._filter_errors(result)
1550 if errors:
1551 error_str = self._get_error_str(errors)
1552 raise LinstorVolumeManagerError(
1553 'Failed to create node interface on `{}`: {}'.format(node_name, error_str)
1554 )
1556 def destroy_node_interface(self, node_name, name):
1557 """
1558 Destroy a node interface in the LINSTOR database.
1559 :param str node_name: Node name of the interface to remove.
1560 :param str name: Interface to remove.
1561 """
1563 if name == 'default':
1564 raise LinstorVolumeManagerError(
1565 'Unable to delete the default interface of a node!'
1566 )
1568 result = self._linstor.netinterface_delete(node_name, name)
1569 errors = self._filter_errors(result)
1570 if errors:
1571 error_str = self._get_error_str(errors)
1572 raise LinstorVolumeManagerError(
1573 'Failed to destroy node interface on `{}`: {}'.format(node_name, error_str)
1574 )
1576 def modify_node_interface(self, node_name, name, ip):
1577 """
1578 Modify a node interface in the LINSTOR database. Create it if necessary.
1579 :param str node_name: Node name of the interface to use.
1580 :param str name: Interface to modify or create.
1581 :param str ip: IP of the interface.
1582 """
1583 result = self._linstor.netinterface_create(node_name, name, ip)
1584 errors = self._filter_errors(result)
1585 if not errors:
1586 return
1588 if self._check_errors(errors, [linstor.consts.FAIL_EXISTS_NET_IF]):
1589 result = self._linstor.netinterface_modify(node_name, name, ip)
1590 errors = self._filter_errors(result)
1591 if not errors:
1592 return
1594 error_str = self._get_error_str(errors)
1595 raise LinstorVolumeManagerError(
1596 'Unable to modify interface on `{}`: {}'.format(node_name, error_str)
1597 )
1599 def list_node_interfaces(self, node_name):
1600 """
1601 List all node interfaces.
1602 :param str node_name: Node name to use to list interfaces.
1603 :rtype: list
1604 :
1605 """
1606 result = self._linstor.net_interface_list(node_name)
1607 if not result:
1608 raise LinstorVolumeManagerError(
1609 'Unable to list interfaces on `{}`: no list received'.format(node_name)
1610 )
1612 interfaces = {}
1613 for interface in result:
1614 interface = interface._rest_data
1615 interfaces[interface['name']] = {
1616 'address': interface['address'],
1617 'active': interface['is_active']
1618 }
1619 return interfaces
1621 def get_node_preferred_interface(self, node_name):
1622 """
1623 Get the preferred interface used by a node.
1624 :param str node_name: Node name of the interface to get.
1625 :rtype: str
1626 """
1627 try:
1628 nodes = self._linstor.node_list_raise([node_name]).nodes
1629 if nodes:
1630 properties = nodes[0].props
1631 return properties.get('PrefNic', 'default')
1632 return nodes
1633 except Exception as e:
1634 raise LinstorVolumeManagerError(
1635 'Failed to get preferred interface: `{}`'.format(e)
1636 )
1638 def set_node_preferred_interface(self, node_name, name):
1639 """
1640 Set the preferred interface to use on a node.
1641 :param str node_name: Node name of the interface.
1642 :param str name: Preferred interface to use.
1643 """
1644 result = self._linstor.node_modify(node_name, property_dict={'PrefNic': name})
1645 errors = self._filter_errors(result)
1646 if errors:
1647 error_str = self._get_error_str(errors)
1648 raise LinstorVolumeManagerError(
1649 'Failed to set preferred node interface on `{}`: {}'.format(node_name, error_str)
1650 )
1652 def get_nodes_info(self):
1653 """
1654 Get all nodes + statuses, used or not by the pool.
1655 :rtype: dict(str, dict)
1656 """
1657 try:
1658 nodes = {}
1659 for node in self._linstor.node_list_raise().nodes:
1660 nodes[node.name] = node.connection_status
1661 return nodes
1662 except Exception as e:
1663 raise LinstorVolumeManagerError(
1664 'Failed to get all nodes: `{}`'.format(e)
1665 )
1667 def get_storage_pools_info(self):
1668 """
1669 Give all storage pools of current group name.
1670 :rtype: dict(str, list)
1671 """
1672 storage_pools = {}
1673 for pool in self._get_storage_pools(force=True):
1674 if pool.node_name not in storage_pools:
1675 storage_pools[pool.node_name] = []
1677 size = -1
1678 capacity = -1
1680 space = pool.free_space
1681 if space:
1682 size = space.free_capacity
1683 if size < 0:
1684 size = -1
1685 else:
1686 size *= 1024
1687 capacity = space.total_capacity
1688 if capacity <= 0:
1689 capacity = -1
1690 else:
1691 capacity *= 1024
1693 storage_pools[pool.node_name].append({
1694 'name': pool.name,
1695 'linstor-uuid': pool.uuid,
1696 'free-size': size,
1697 'capacity': capacity
1698 })
1700 return storage_pools
1702 def get_resources_info(self):
1703 """
1704 Give all resources of current group name.
1705 :rtype: dict(str, list)
1706 """
1707 if self._resources_info_cache and not self._resource_cache_dirty:
1708 return self._resources_info_cache
1710 resources = {}
1711 resource_list = self._get_resource_cache()
1712 volume_names = self.get_volumes_with_name()
1713 for resource in resource_list.resources:
1714 if resource.name not in resources:
1715 resources[resource.name] = { 'nodes': {}, 'uuid': '' }
1716 resource_nodes = resources[resource.name]['nodes']
1718 resource_nodes[resource.node_name] = {
1719 'volumes': [],
1720 'diskful': linstor.consts.FLAG_DISKLESS not in resource.flags,
1721 'tie-breaker': linstor.consts.FLAG_TIE_BREAKER in resource.flags
1722 }
1723 resource_volumes = resource_nodes[resource.node_name]['volumes']
1725 for volume in resource.volumes:
1726 # We ignore diskless pools of the form "DfltDisklessStorPool".
1727 if volume.storage_pool_name != self._group_name:
1728 continue
1730 usable_size = volume.usable_size
1731 if usable_size < 0:
1732 usable_size = -1
1733 else:
1734 usable_size *= 1024
1736 allocated_size = volume.allocated_size
1737 if allocated_size < 0:
1738 allocated_size = -1
1739 else:
1740 allocated_size *= 1024
1742 resource_volumes.append({
1743 'storage-pool-name': volume.storage_pool_name,
1744 'linstor-uuid': volume.uuid,
1745 'number': volume.number,
1746 'device-path': volume.device_path,
1747 'usable-size': usable_size,
1748 'allocated-size': allocated_size
1749 })
1751 for resource_state in resource_list.resource_states:
1752 resource = resources[resource_state.rsc_name]['nodes'][resource_state.node_name]
1753 resource['in-use'] = resource_state.in_use
1755 volumes = resource['volumes']
1756 for volume_state in resource_state.volume_states:
1757 volume = next((x for x in volumes if x['number'] == volume_state.number), None)
1758 if volume:
1759 volume['disk-state'] = volume_state.disk_state
1761 for volume_uuid, volume_name in volume_names.items():
1762 resource = resources.get(volume_name)
1763 if resource:
1764 resource['uuid'] = volume_uuid
1766 self._resources_info_cache = resources
1767 return self._resources_info_cache
1769 def get_resource_info(self, volume_uuid: str) -> Dict[str, Any]:
1770 """
1771 Give a resource info based on its UUID.
1772 :param volume_uuid str: volume uuid to search for
1773 :rtype: dict(str, any)
1774 """
1775 for volume in self.get_resources_info().values():
1776 if volume["uuid"] == volume_uuid:
1777 return volume
1779 raise LinstorVolumeManagerError(
1780 f"Could not find info about volume `{volume_uuid}`",
1781 LinstorVolumeManagerError.ERR_VOLUME_NOT_EXISTS
1782 )
1784 def get_database_path(self):
1785 """
1786 Get the database path.
1787 :return: The current database path.
1788 :rtype: str
1789 """
1790 return self._request_database_path(self._linstor, activate=True)
1792 @classmethod
1793 def is_controller(cls):
1794 return cls._is_mounted(DATABASE_PATH)
1796 @classmethod
1797 def get_database_backup_age(cls):
1798 """
1799 Return the latest backup age in seconds.
1800 If not called on the Controller, since backups are not available,
1801 returns a huge value (a timestamp of now).
1802 """
1803 return (datetime.now() - cls._get_latest_database_backup()[1]).total_seconds()
1805 def database_backup(self, name=""):
1806 # Create new backup
1807 date = datetime.now().strftime(DATABASE_BACKUP_DATE_FORMAT)
1808 filename = DATABASE_BACKUP_NAME_FORMAT.format(date, name)
1809 self._linstor.controller_backupdb(filename)
1810 # Relative path are ok for a secondary backup filename:
1811 # https://github.com/LINBIT/linstor-server/blob/3e9306a9d8215606544c64c50ced150625ee4926/controller/src/main/java/com/linbit/linstor/api/rest/v1/Controller.java#L408
1812 self._linstor.controller_backupdb(f"../linstor.d/db-backups/{filename}")
1813 util.SMlog(f"[database_backup] Created: {filename}", priority=util.LOG_INFO)
1815 @classmethod
1816 def database_backup_validate_and_prune(cls):
1817 """
1818 Removes old backup based on two criterias:
1819 - Validity of the zipfile done by self._check_database_backup.
1820 - Number of valid files found, only the nth latest are kept.
1821 """
1822 for directory in (DATABASE_BACKUP_DIR_MAIN, DATABASE_BACKUP_DIR_SPARE):
1823 valid_backup_count = 0
1824 # Validate file and apply retention
1825 for database_backup_path, _ in cls._get_sorted_database_backup(directory):
1826 try:
1827 cls._check_database_backup(database_backup_path)
1828 valid_backup_count += 1
1829 if valid_backup_count < DATABASE_BACKUP_RETENTION:
1830 continue
1831 except LinstorDatabaseBackupError as error:
1832 util.SMlog(f"[database_backup] Check failed `{error}` [{database_backup_path}]",
1833 priority=util.LOG_ERR)
1834 with contextlib.suppress(OSError):
1835 os.unlink(database_backup_path)
1837 @classmethod
1838 def get_all_group_names(cls, base_name):
1839 """
1840 Get all group names. I.e. list of current group + HA.
1841 :param str base_name: The SR group_name to use.
1842 :return: List of group names.
1843 :rtype: list
1844 """
1845 return [cls.build_group_name(base_name), cls._build_ha_group_name(base_name)]
1847 @classmethod
1848 def get_volume_group_name(cls, volume_name) -> str:
1849 """
1850 Get the group name associated with a volume.
1851 :param str volume_name: The volume name to find.
1852 :return: A group name.
1853 :rtype: str
1854 """
1856 lin = cls._create_linstor_instance(uri=None)
1857 try:
1858 for dfn in lin.resource_dfn_list_raise(
1859 query_volume_definitions=False,
1860 filter_by_resource_definitions=[volume_name]
1861 ).resource_definitions:
1862 return dfn.resource_group_name
1863 finally:
1864 lin.disconnect()
1866 return ''
1868 @classmethod
1869 def create_sr(cls, group_name, ips, redundancy, thin_provisioning, logger=default_logger.__func__):
1870 """
1871 Create a new SR on the given nodes.
1872 :param str group_name: The SR group_name to use.
1873 :param set(str) ips: Node ips.
1874 :param int redundancy: How many copy of volumes should we store?
1875 :param bool thin_provisioning: Use thin or thick provisioning.
1876 :param function logger: Function to log messages.
1877 :return: A new LinstorSr instance.
1878 :rtype: LinstorSr
1879 """
1881 try:
1882 cls._start_controller(start=True)
1883 sr = cls._create_sr(group_name, ips, redundancy, thin_provisioning, logger)
1884 finally:
1885 # Controller must be stopped and volume unmounted because
1886 # it is the role of the drbd-reactor daemon to do the right
1887 # actions.
1888 cls._start_controller(start=False)
1889 cls._mount_volume(
1890 cls.build_device_path(DATABASE_VOLUME_NAME),
1891 DATABASE_PATH,
1892 mount=False
1893 )
1894 return sr
1896 @classmethod
1897 def _create_sr(cls, group_name, ips, redundancy, thin_provisioning, logger=default_logger.__func__):
1898 # 1. Check if SR already exists.
1899 uri = 'linstor://localhost'
1901 lin = cls._create_linstor_instance(uri, keep_uri_unmodified=True)
1903 node_names = list(ips.keys())
1904 for node_name, ip in ips.items():
1905 while True:
1906 # Try to create node.
1907 result = lin.node_create(
1908 node_name,
1909 linstor.consts.VAL_NODE_TYPE_CMBD,
1910 ip
1911 )
1913 errors = cls._filter_errors(result)
1914 if cls._check_errors(
1915 errors, [linstor.consts.FAIL_EXISTS_NODE]
1916 ):
1917 # If it already exists, remove, then recreate.
1918 result = lin.node_delete(node_name)
1919 error_str = cls._get_error_str(result)
1920 if error_str:
1921 raise LinstorVolumeManagerError(
1922 'Failed to remove old node `{}`: {}'
1923 .format(node_name, error_str)
1924 )
1925 elif not errors:
1926 break # Created!
1927 else:
1928 raise LinstorVolumeManagerError(
1929 'Failed to create node `{}` with ip `{}`: {}'.format(
1930 node_name, ip, cls._get_error_str(errors)
1931 )
1932 )
1934 driver_pool_name = group_name
1935 base_group_name = group_name
1936 group_name = cls.build_group_name(group_name)
1937 storage_pool_name = group_name
1938 pools = lin.storage_pool_list_raise(filter_by_stor_pools=[storage_pool_name]).storage_pools
1939 if pools:
1940 existing_node_names = [pool.node_name for pool in pools]
1941 raise LinstorVolumeManagerError(
1942 'Unable to create SR `{}`. It already exists on node(s): {}'
1943 .format(group_name, existing_node_names)
1944 )
1946 if lin.resource_group_list_raise(
1947 cls.get_all_group_names(base_group_name)
1948 ).resource_groups:
1949 if not lin.resource_dfn_list_raise().resource_definitions:
1950 backup_path = cls._create_database_backup_path()
1951 logger(
1952 'Group name already exists `{}` without LVs. '
1953 'Ignoring and moving the config files in {}'.format(group_name, backup_path)
1954 )
1955 cls._move_files(DATABASE_PATH, backup_path)
1956 else:
1957 raise LinstorVolumeManagerError(
1958 'Unable to create SR `{}`: The group name already exists'
1959 .format(group_name)
1960 )
1962 if thin_provisioning:
1963 driver_pool_parts = driver_pool_name.split('/')
1964 if not len(driver_pool_parts) == 2:
1965 raise LinstorVolumeManagerError(
1966 'Invalid group name using thin provisioning. '
1967 'Expected format: \'VG/LV`\''
1968 )
1970 # 2. Create storage pool on each node + resource group.
1971 reg_volume_group_not_found = re.compile(
1972 ".*Volume group '.*' not found$"
1973 )
1975 i = 0
1976 try:
1977 # 2.a. Create storage pools.
1978 storage_pool_count = 0
1979 while i < len(node_names):
1980 node_name = node_names[i]
1982 result = lin.storage_pool_create(
1983 node_name=node_name,
1984 storage_pool_name=storage_pool_name,
1985 storage_driver='LVM_THIN' if thin_provisioning else 'LVM',
1986 driver_pool_name=driver_pool_name
1987 )
1989 errors = linstor.Linstor.filter_api_call_response_errors(
1990 result
1991 )
1992 if errors:
1993 if errors[0].is_error(
1994 linstor.consts.FAIL_STOR_POOL_CONFIGURATION_ERROR
1995 ) and reg_volume_group_not_found.match(errors[0].message):
1996 logger(
1997 'Volume group `{}` not found on `{}`. Ignoring...'
1998 .format(group_name, node_name)
1999 )
2000 cls._destroy_storage_pool(lin, storage_pool_name, node_name)
2001 else:
2002 error_str = cls._get_error_str(result)
2003 raise LinstorVolumeManagerError(
2004 'Could not create SP `{}` on node `{}`: {}'
2005 .format(group_name, node_name, error_str)
2006 )
2007 else:
2008 storage_pool_count += 1
2009 i += 1
2011 if not storage_pool_count:
2012 raise LinstorVolumeManagerError(
2013 'Unable to create SR `{}`: No VG group found'.format(
2014 group_name,
2015 )
2016 )
2018 # 2.b. Create resource groups.
2019 ha_group_name = cls._build_ha_group_name(base_group_name)
2020 cls._create_resource_group(
2021 lin,
2022 group_name,
2023 storage_pool_name,
2024 redundancy,
2025 True
2026 )
2027 cls._create_resource_group(
2028 lin,
2029 ha_group_name,
2030 storage_pool_name,
2031 3,
2032 True
2033 )
2035 # 3. Create the LINSTOR database volume and mount it.
2036 try:
2037 logger('Creating database volume...')
2038 volume_path = cls._create_database_volume(
2039 lin, ha_group_name, storage_pool_name, node_names, redundancy
2040 )
2041 except LinstorVolumeManagerError as e:
2042 if e.code != LinstorVolumeManagerError.ERR_VOLUME_EXISTS:
2043 logger('Destroying database volume after creation fail...')
2044 cls._force_destroy_database_volume(lin, group_name)
2045 raise
2047 try:
2048 logger('Mounting database volume...')
2050 # First we must disable the controller to move safely the
2051 # LINSTOR config.
2052 cls._start_controller(start=False)
2054 cls._mount_database_volume(volume_path)
2055 except Exception as e:
2056 # Ensure we are connected because controller has been
2057 # restarted during mount call.
2058 logger('Destroying database volume after mount fail...')
2060 try:
2061 cls._start_controller(start=True)
2062 except Exception:
2063 pass
2065 lin = cls._create_linstor_instance(
2066 uri, keep_uri_unmodified=True
2067 )
2068 cls._force_destroy_database_volume(lin, group_name)
2069 raise e
2071 cls._start_controller(start=True)
2072 lin = cls._create_linstor_instance(uri, keep_uri_unmodified=True)
2074 # 4. Remove storage pools/resource/volume group in the case of errors.
2075 except Exception as e:
2076 logger('Destroying resource group and storage pools after fail...')
2077 try:
2078 cls._destroy_resource_group(lin, group_name)
2079 cls._destroy_resource_group(lin, ha_group_name)
2080 except Exception as e2:
2081 logger('Failed to destroy resource group: {}'.format(e2))
2082 pass
2083 j = 0
2084 i = min(i, len(node_names) - 1)
2085 while j <= i:
2086 try:
2087 cls._destroy_storage_pool(lin, storage_pool_name, node_names[j])
2088 except Exception as e2:
2089 logger('Failed to destroy resource group: {}'.format(e2))
2090 pass
2091 j += 1
2092 raise e
2094 # 5. Return new instance.
2095 instance = cls.__new__(cls)
2096 instance._linstor = lin
2097 instance._logger = logger
2098 instance._redundancy = redundancy
2099 instance._base_group_name = base_group_name
2100 instance._group_name = group_name
2101 instance._volumes = set()
2102 instance._storage_pools_time = 0
2103 instance._kv_cache = instance._create_kv_cache()
2104 instance._resource_cache = None
2105 instance._resource_cache_dirty = True
2106 instance._volume_info_cache = None
2107 instance._volume_info_cache_dirty = True
2108 return instance
2110 @classmethod
2111 def build_device_path(cls, volume_name):
2112 """
2113 Build a device path given a volume name.
2114 :param str volume_name: The volume name to use.
2115 :return: A valid or not device path.
2116 :rtype: str
2117 """
2119 return '{}{}/0'.format(cls.DEV_ROOT_PATH, volume_name)
2121 @classmethod
2122 def build_volume_name(cls, base_name):
2123 """
2124 Build a volume name given a base name (i.e. a UUID).
2125 :param str base_name: The volume name to use.
2126 :return: A valid or not device path.
2127 :rtype: str
2128 """
2129 return '{}{}'.format(cls.PREFIX_VOLUME, base_name)
2131 @classmethod
2132 def round_up_volume_size(cls, volume_size):
2133 """
2134 Align volume size on higher multiple of BLOCK_SIZE.
2135 :param int volume_size: The volume size to align.
2136 :return: An aligned volume size.
2137 :rtype: int
2138 """
2139 return round_up(volume_size, cls.BLOCK_SIZE)
2141 @classmethod
2142 def round_down_volume_size(cls, volume_size):
2143 """
2144 Align volume size on lower multiple of BLOCK_SIZE.
2145 :param int volume_size: The volume size to align.
2146 :return: An aligned volume size.
2147 :rtype: int
2148 """
2149 return round_down(volume_size, cls.BLOCK_SIZE)
2151 # --------------------------------------------------------------------------
2152 # Private helpers.
2153 # --------------------------------------------------------------------------
2155 def _create_kv_cache(self):
2156 self._kv_cache = self._create_linstor_kv('/')
2157 self._kv_cache_dirty = False
2158 return self._kv_cache
2160 def _get_kv_cache(self):
2161 if self._kv_cache_dirty:
2162 self._kv_cache = self._create_kv_cache()
2163 return self._kv_cache
2165 def _create_resource_cache(self):
2166 self._resource_cache = self._linstor.resource_list_raise()
2167 self._resource_cache_dirty = False
2168 return self._resource_cache
2170 def _get_resource_cache(self):
2171 if self._resource_cache_dirty:
2172 self._resource_cache = self._create_resource_cache()
2173 return self._resource_cache
2175 def _mark_resource_cache_as_dirty(self):
2176 self._resource_cache_dirty = True
2177 self._volume_info_cache_dirty = True
2179 # --------------------------------------------------------------------------
2181 def _ensure_volume_exists(self, volume_uuid):
2182 if volume_uuid not in self._volumes:
2183 raise LinstorVolumeManagerError(
2184 'volume `{}` doesn\'t exist'.format(volume_uuid),
2185 LinstorVolumeManagerError.ERR_VOLUME_NOT_EXISTS
2186 )
2188 def _find_best_size_candidates(self):
2189 result = self._linstor.resource_group_qmvs(self._group_name)
2190 error_str = self._get_error_str(result)
2191 if error_str:
2192 raise LinstorVolumeManagerError(
2193 'Failed to get max volume size allowed of SR `{}`: {}'.format(
2194 self._group_name,
2195 error_str
2196 )
2197 )
2198 return result[0].candidates
2200 def _fetch_resource_names(self, ignore_deleted=True):
2201 resource_names = set()
2202 dfns = self._linstor.resource_dfn_list_raise().resource_definitions
2203 for dfn in dfns:
2204 if dfn.resource_group_name in self.get_all_group_names(self._base_group_name) and (
2205 ignore_deleted or
2206 linstor.consts.FLAG_DELETE not in dfn.flags
2207 ):
2208 resource_names.add(dfn.name)
2209 return resource_names
2211 def _get_volumes_info(self, volume_names=None):
2212 all_volume_info = {}
2214 if not self._volume_info_cache_dirty:
2215 return self._volume_info_cache
2217 # `volume_names` MUST contain all volumes registered in the KV store.
2218 # It can be provided to the function to avoid double fetching.
2219 if not volume_names:
2220 volume_names = self.get_volumes_with_name()
2221 volume_names = set(volume_names.values())
2223 def process_resource(resource):
2224 if resource.name not in all_volume_info:
2225 current = all_volume_info[resource.name] = self.VolumeInfo(
2226 resource.name
2227 )
2228 else:
2229 current = all_volume_info[resource.name]
2231 if linstor.consts.FLAG_DISKLESS not in resource.flags:
2232 current.diskful.append(resource.node_name)
2234 for volume in resource.volumes:
2235 # We ignore diskless pools of the form "DfltDisklessStorPool".
2236 if volume.storage_pool_name != self._group_name:
2237 continue
2238 # Only fetch first volume.
2239 if volume.number != 0:
2240 continue
2242 allocated_size = volume.allocated_size
2243 if allocated_size > current.allocated_size:
2244 current.allocated_size = allocated_size
2246 usable_size = volume.usable_size
2247 if usable_size > 0 and (
2248 usable_size < current.virtual_size or
2249 not current.virtual_size
2250 ):
2251 current.virtual_size = usable_size
2253 try:
2254 for resource in self._get_resource_cache().resources:
2255 if resource.name in volume_names:
2256 process_resource(resource)
2257 for volume in all_volume_info.values():
2258 if volume.allocated_size <= 0:
2259 raise LinstorVolumeManagerError('Failed to get allocated size of `{}`'.format(resource.name))
2261 if volume.virtual_size <= 0:
2262 raise LinstorVolumeManagerError('Failed to get usable size of `{}`'.format(volume.name))
2264 volume.allocated_size *= 1024
2265 volume.virtual_size *= 1024
2266 except LinstorVolumeManagerError:
2267 self._mark_resource_cache_as_dirty()
2268 raise
2270 self._volume_info_cache_dirty = False
2271 self._volume_info_cache = all_volume_info
2273 return all_volume_info
2275 def _get_volume_node_names_and_size(self, volume_name):
2276 node_names = set()
2277 size = -1
2278 for resource in self._linstor.resource_list_raise(
2279 filter_by_resources=[volume_name]
2280 ).resources:
2281 for volume in resource.volumes:
2282 # We ignore diskless pools of the form "DfltDisklessStorPool".
2283 if volume.storage_pool_name != self._group_name:
2284 continue
2286 node_names.add(resource.node_name)
2288 usable_size = volume.usable_size
2289 if usable_size <= 0:
2290 continue
2292 if size < 0:
2293 size = usable_size
2294 else:
2295 size = min(size, usable_size)
2297 if size <= 0:
2298 raise LinstorVolumeManagerError('Failed to get usable size of `{}`'.format(resource.name))
2300 return (node_names, size * 1024)
2302 def _compute_size(self, attr):
2303 capacity = 0
2304 for pool in self._get_storage_pools(force=True):
2305 space = pool.free_space
2306 if space:
2307 size = getattr(space, attr)
2308 if size < 0:
2309 raise LinstorVolumeManagerError(
2310 'Failed to get pool {} attr of `{}`'
2311 .format(attr, pool.node_name)
2312 )
2313 capacity += size
2314 return capacity * 1024
2316 def _get_node_names(self):
2317 node_names = set()
2318 for pool in self._get_storage_pools():
2319 node_names.add(pool.node_name)
2320 return node_names
2322 def _get_storage_pools(self, force=False):
2323 cur_time = time.time()
2324 elsaped_time = cur_time - self._storage_pools_time
2326 if force or elsaped_time >= self.STORAGE_POOLS_FETCH_INTERVAL:
2327 self._storage_pools = self._linstor.storage_pool_list_raise(
2328 filter_by_stor_pools=[self._group_name]
2329 ).storage_pools
2330 self._storage_pools_time = time.time()
2332 return self._storage_pools
2334 def _create_volume(
2335 self,
2336 volume_uuid,
2337 volume_name,
2338 size,
2339 place_resources,
2340 high_availability
2341 ):
2342 size = self.round_up_volume_size(size)
2343 self._mark_resource_cache_as_dirty()
2345 group_name = self._ha_group_name if high_availability else self._group_name
2346 def create_definition():
2347 first_attempt = True
2348 while True:
2349 try:
2350 self._check_volume_creation_errors(
2351 self._linstor.resource_group_spawn(
2352 rsc_grp_name=group_name,
2353 rsc_dfn_name=volume_name,
2354 vlm_sizes=['{}B'.format(size)],
2355 definitions_only=True
2356 ),
2357 volume_uuid,
2358 self._group_name
2359 )
2360 break
2361 except LinstorVolumeManagerError as e:
2362 if (
2363 not first_attempt or
2364 not high_availability or
2365 e.code != LinstorVolumeManagerError.ERR_GROUP_NOT_EXISTS
2366 ):
2367 raise
2369 first_attempt = False
2370 self._create_resource_group(
2371 self._linstor,
2372 group_name,
2373 self._group_name,
2374 3,
2375 True
2376 )
2378 self._configure_volume_peer_slots(self._linstor, volume_name)
2380 def clean():
2381 try:
2382 self._destroy_volume(volume_uuid, force=True, preserve_properties=True)
2383 except Exception as e:
2384 self._logger(
2385 'Unable to destroy volume {} after creation fail: {}'
2386 .format(volume_uuid, e)
2387 )
2389 def create():
2390 try:
2391 create_definition()
2392 if place_resources:
2393 # Basic case when we use the default redundancy of the group.
2394 self._check_volume_creation_errors(
2395 self._linstor.resource_auto_place(
2396 rsc_name=volume_name,
2397 place_count=self._redundancy,
2398 diskless_on_remaining=False
2399 ),
2400 volume_uuid,
2401 self._group_name
2402 )
2403 except LinstorVolumeManagerError as e:
2404 if e.code != LinstorVolumeManagerError.ERR_VOLUME_EXISTS:
2405 clean()
2406 raise
2407 except Exception:
2408 clean()
2409 raise
2411 util.retry(create, maxretry=5)
2413 def _create_volume_with_properties(
2414 self,
2415 volume_uuid,
2416 volume_name,
2417 size,
2418 place_resources,
2419 high_availability
2420 ):
2421 if self.check_volume_exists(volume_uuid):
2422 raise LinstorVolumeManagerError(
2423 'Could not create volume `{}` from SR `{}`, it already exists'
2424 .format(volume_uuid, self._group_name) + ' in properties',
2425 LinstorVolumeManagerError.ERR_VOLUME_EXISTS
2426 )
2428 if volume_name in self._fetch_resource_names():
2429 raise LinstorVolumeManagerError(
2430 'Could not create volume `{}` from SR `{}`, '.format(
2431 volume_uuid, self._group_name
2432 ) + 'resource of the same name already exists in LINSTOR'
2433 )
2435 # I am paranoid.
2436 volume_properties = self._get_volume_properties(volume_uuid)
2437 if (volume_properties.get(self.PROP_NOT_EXISTS) is not None):
2438 raise LinstorVolumeManagerError(
2439 'Could not create volume `{}`, '.format(volume_uuid) +
2440 'properties already exist'
2441 )
2443 try:
2444 volume_properties[self.PROP_NOT_EXISTS] = self.STATE_CREATING
2445 volume_properties[self.PROP_VOLUME_NAME] = volume_name
2447 self._create_volume(
2448 volume_uuid,
2449 volume_name,
2450 size,
2451 place_resources,
2452 high_availability
2453 )
2455 assert volume_properties.namespace == \
2456 self._build_volume_namespace(volume_uuid)
2457 return volume_properties
2458 except LinstorVolumeManagerError as e:
2459 # Do not destroy existing resource!
2460 # In theory we can't get this error because we check this event
2461 # before the `self._create_volume` case.
2462 # It can only happen if the same volume uuid is used in the same
2463 # call in another host.
2464 if e.code != LinstorVolumeManagerError.ERR_VOLUME_EXISTS:
2465 self._destroy_volume(volume_uuid, force=True)
2466 raise
2468 def _find_device_path(self, volume_uuid, volume_name):
2469 current_device_path = self._request_device_path(
2470 volume_uuid, volume_name, activate=True
2471 )
2473 # We use realpath here to get the /dev/drbd<id> path instead of
2474 # /dev/drbd/by-res/<resource_name>.
2475 expected_device_path = self.build_device_path(volume_name)
2476 util.wait_for_path(expected_device_path, 5)
2478 device_realpath = os.path.realpath(expected_device_path)
2479 if current_device_path != device_realpath:
2480 raise LinstorVolumeManagerError(
2481 'Invalid path, current={}, expected={} (realpath={})'
2482 .format(
2483 current_device_path,
2484 expected_device_path,
2485 device_realpath
2486 )
2487 )
2488 return expected_device_path
2490 def _request_device_path(self, volume_uuid, volume_name, activate=False):
2491 node_name = socket.gethostname()
2493 resource = next(filter(
2494 lambda resource: resource.node_name == node_name and
2495 resource.name == volume_name,
2496 self._get_resource_cache().resources
2497 ), None)
2499 if not resource:
2500 if activate:
2501 self._mark_resource_cache_as_dirty()
2502 self._activate_device_path(
2503 self._linstor, node_name, volume_name
2504 )
2505 return self._request_device_path(volume_uuid, volume_name)
2506 raise LinstorVolumeManagerError(
2507 'Unable to get dev path for `{}`, no resource found but definition "seems" to exist'
2508 .format(volume_uuid)
2509 )
2511 # Contains a path of the /dev/drbd<id> form.
2512 device_path = resource.volumes[0].device_path
2513 if not device_path:
2514 raise LinstorVolumeManagerError('Empty dev path for `{}`!'.format(volume_uuid))
2515 return device_path
2517 def _destroy_resource(self, resource_name, force=False):
2518 result = self._linstor.resource_dfn_delete(resource_name)
2519 error_str = self._get_error_str(result)
2520 if not error_str:
2521 self._mark_resource_cache_as_dirty()
2522 return
2524 if not force:
2525 self._mark_resource_cache_as_dirty()
2526 raise LinstorVolumeManagerError(
2527 'Could not destroy resource `{}` from SR `{}`: {}'
2528 .format(resource_name, self._group_name, error_str)
2529 )
2531 # If force is used, ensure there is no opener.
2532 openers = get_all_volume_openers(resource_name, '0')
2533 for host_openers in openers.values():
2534 if host_openers:
2535 self._mark_resource_cache_as_dirty()
2536 raise LinstorVolumeManagerError(
2537 'Could not force destroy resource `{}` from SR `{}`: {} (openers=`{}`)'
2538 .format(resource_name, self._group_name, error_str, openers)
2539 )
2541 # Maybe the resource is blocked in primary mode. DRBD/LINSTOR issue?
2542 resource_states = filter(
2543 lambda resource_state: resource_state.name == resource_name,
2544 self._get_resource_cache().resource_states
2545 )
2547 # Mark only after computation of states.
2548 self._mark_resource_cache_as_dirty()
2550 for resource_state in resource_states:
2551 volume_state = resource_state.volume_states[0]
2552 if resource_state.in_use:
2553 demote_drbd_resource(resource_state.node_name, resource_name)
2554 break
2555 self._destroy_resource(resource_name)
2557 def _destroy_volume(self, volume_uuid, force=False, preserve_properties=False):
2558 volume_properties = self._get_volume_properties(volume_uuid)
2559 try:
2560 volume_name = volume_properties.get(self.PROP_VOLUME_NAME)
2561 if volume_name in self._fetch_resource_names():
2562 self._destroy_resource(volume_name, force)
2564 # Assume this call is atomic.
2565 if not preserve_properties:
2566 volume_properties.clear()
2567 except Exception as e:
2568 raise LinstorVolumeManagerError(
2569 'Cannot destroy volume `{}`: {}'.format(volume_uuid, e)
2570 )
2572 def _build_volumes(self, repair):
2573 properties = self._kv_cache
2574 resource_names = self._fetch_resource_names()
2576 self._volumes = set()
2578 updating_uuid_volumes = self._get_volumes_by_property(
2579 self.REG_UPDATING_UUID_SRC, ignore_inexisting_volumes=False
2580 )
2581 if updating_uuid_volumes and not repair:
2582 raise LinstorVolumeManagerError(
2583 'Cannot build LINSTOR volume list: '
2584 'It exists invalid "updating uuid volumes", repair is required'
2585 )
2587 existing_volumes = self._get_volumes_by_property(
2588 self.REG_NOT_EXISTS, ignore_inexisting_volumes=False
2589 )
2590 for volume_uuid, not_exists in existing_volumes.items():
2591 properties.namespace = self._build_volume_namespace(volume_uuid)
2593 src_uuid = properties.get(self.PROP_UPDATING_UUID_SRC)
2594 if src_uuid:
2595 self._logger(
2596 'Ignoring volume during manager initialization with prop '
2597 ' PROP_UPDATING_UUID_SRC: {} (properties={})'
2598 .format(
2599 volume_uuid,
2600 self._get_filtered_properties(properties)
2601 )
2602 )
2603 continue
2605 # Insert volume in list if the volume exists. Or if the volume
2606 # is being created and a slave wants to use it (repair = False).
2607 #
2608 # If we are on the master and if repair is True and state is
2609 # Creating, it's probably a bug or crash: the creation process has
2610 # been stopped.
2611 if not_exists == self.STATE_EXISTS or (
2612 not repair and not_exists == self.STATE_CREATING
2613 ):
2614 self._volumes.add(volume_uuid)
2615 continue
2617 if not repair:
2618 self._logger(
2619 'Ignoring bad volume during manager initialization: {} '
2620 '(properties={})'.format(
2621 volume_uuid,
2622 self._get_filtered_properties(properties)
2623 )
2624 )
2625 continue
2627 # Remove bad volume.
2628 try:
2629 self._logger(
2630 'Removing bad volume during manager initialization: {} '
2631 '(properties={})'.format(
2632 volume_uuid,
2633 self._get_filtered_properties(properties)
2634 )
2635 )
2636 volume_name = properties.get(self.PROP_VOLUME_NAME)
2638 # Little optimization, don't call `self._destroy_volume`,
2639 # we already have resource name list.
2640 if volume_name in resource_names:
2641 self._destroy_resource(volume_name, force=True)
2643 # Assume this call is atomic.
2644 properties.clear()
2645 except Exception as e:
2646 # Do not raise, we don't want to block user action.
2647 self._logger(
2648 'Cannot clean volume {}: {}'.format(volume_uuid, e)
2649 )
2651 # The volume can't be removed, maybe it's still in use,
2652 # in this case rename it with the "DELETED_" prefix.
2653 # This prefix is mandatory if it exists a snap transaction to
2654 # rollback because the original VDI UUID can try to be renamed
2655 # with the UUID we are trying to delete...
2656 if not volume_uuid.startswith('DELETED_'):
2657 self.update_volume_uuid(
2658 volume_uuid, 'DELETED_' + volume_uuid, force=True
2659 )
2661 for dest_uuid, src_uuid in updating_uuid_volumes.items():
2662 dest_namespace = self._build_volume_namespace(dest_uuid)
2664 properties.namespace = dest_namespace
2665 if int(properties.get(self.PROP_NOT_EXISTS)):
2666 properties.clear()
2667 continue
2669 properties.namespace = self._build_volume_namespace(src_uuid)
2670 properties.clear()
2672 properties.namespace = dest_namespace
2673 properties.pop(self.PROP_UPDATING_UUID_SRC)
2675 if src_uuid in self._volumes:
2676 self._volumes.remove(src_uuid)
2677 self._volumes.add(dest_uuid)
2679 def _get_sr_properties(self):
2680 return self._create_linstor_kv(self._build_sr_namespace())
2682 def _get_volumes_by_property(
2683 self, reg_prop, ignore_inexisting_volumes=True
2684 ):
2685 base_properties = self._get_kv_cache()
2686 base_properties.namespace = self._build_volume_namespace()
2688 volume_properties = {}
2689 for volume_uuid in self._volumes:
2690 volume_properties[volume_uuid] = ''
2692 for key, value in base_properties.items():
2693 res = reg_prop.match(key)
2694 if res:
2695 volume_uuid = res.groups()[0]
2696 if not ignore_inexisting_volumes or \
2697 volume_uuid in self._volumes:
2698 volume_properties[volume_uuid] = value
2700 return volume_properties
2702 def _create_linstor_kv(self, namespace):
2703 return linstor.KV(
2704 self._group_name,
2705 uri=self._linstor.controller_host(),
2706 namespace=namespace
2707 )
2709 def _get_volume_properties(self, volume_uuid):
2710 properties = self._get_kv_cache()
2711 properties.namespace = self._build_volume_namespace(volume_uuid)
2712 return properties
2714 @classmethod
2715 def _list_database_backups(cls, database_backup_dir):
2716 """
2717 List all visible backup files in database_backup_dir.
2718 DATABASE_BACKUP_DIR_MAIN is only available on the Linstor Controller.
2719 DATABASE_BACKUP_DIR_SPARE will list backups previously made when the Host was the Linstor Controller.
2720 This may not be useful information if it is not the Controller anymore.
2721 """
2722 for path in database_backup_dir.glob(DATABASE_BACKUP_NAME_FORMAT.format(
2723 "[0-9]" * 8 + "_" + "[0-9]" * 6, "*") + ".zip"):
2724 try:
2725 yield path, datetime.strptime(path.name.split("-")[1], DATABASE_BACKUP_DATE_FORMAT)
2726 except (ValueError, IndexError):
2727 continue
2729 @classmethod
2730 def _get_sorted_database_backup(cls, database_backup_dir):
2731 """
2732 Return list of backups in database_backup_dir, alongside their creation date.
2733 Sorted by date from the more recent to the older one.
2734 """
2735 return sorted(cls._list_database_backups(database_backup_dir),
2736 reverse=True,
2737 key=lambda p: p[1])
2739 @classmethod
2740 def _get_latest_database_backup(cls):
2741 """
2742 Return the latest backup in DATABASE_BACKUP_DIR_MAIN, and its creation date.
2743 Returns (None, timestamp(0)) when none are found.
2744 None will be found if it is not called on the Linstor Controller.
2745 (cf _list_database_backups)
2746 """
2747 return max(cls._list_database_backups(DATABASE_BACKUP_DIR_MAIN),
2748 default=(None, datetime.fromtimestamp(0)),
2749 key=lambda p: p[1])
2751 @classmethod
2752 def _check_database_backup(cls, database_backup_path):
2753 """
2754 Make some validation of a database backup zip-file.
2755 Check its a valid zipfile, and CRC-test its content.
2756 Check it contains a non-empty linstordb.mv.db file.
2757 Always raises a LinstorDatabaseBackupError if checks failed.
2758 """
2759 try:
2760 with zipfile.ZipFile(database_backup_path, mode="r") as archive:
2761 if archive.testzip() is not None:
2762 raise LinstorDatabaseBackupError("zip archive CRC failed")
2763 linstordb = next((
2764 f
2765 for f in archive.filelist
2766 if f.filename == "linstordb.mv.db"
2767 ), None)
2768 if not linstordb:
2769 raise LinstorDatabaseBackupError("cannot find linstordb.mv.db")
2770 if linstordb.file_size == 0:
2771 raise LinstorDatabaseBackupError("linstordb.mv.db is empty")
2772 except (FileNotFoundError, zipfile.BadZipFile, zipfile.LargeZipFile) as e:
2773 raise LinstorDatabaseBackupError(e) from e
2775 @classmethod
2776 def _build_sr_namespace(cls):
2777 return '/{}/'.format(cls.NAMESPACE_SR)
2779 @classmethod
2780 def _build_volume_namespace(cls, volume_uuid=None):
2781 # Return a path to all volumes if `volume_uuid` is not given.
2782 if volume_uuid is None:
2783 return '/{}/'.format(cls.NAMESPACE_VOLUME)
2784 return '/{}/{}/'.format(cls.NAMESPACE_VOLUME, volume_uuid)
2786 @classmethod
2787 def _get_error_str(cls, result):
2788 return ', '.join([
2789 err.message for err in cls._filter_errors(result)
2790 ])
2792 @classmethod
2793 def _create_linstor_instance(
2794 cls, uri, keep_uri_unmodified=False, attempt_count=30
2795 ):
2796 retry = False
2798 def connect(uri):
2799 if not uri:
2800 uri = get_controller_uri()
2801 if not uri:
2802 raise LinstorVolumeManagerError(
2803 'Unable to find controller uri...'
2804 )
2805 instance = linstor.Linstor(uri, keep_alive=True)
2806 instance.connect()
2807 return instance
2809 try:
2810 return connect(uri)
2811 except (linstor.errors.LinstorNetworkError, LinstorVolumeManagerError):
2812 pass
2814 if not keep_uri_unmodified:
2815 uri = None
2817 return util.retry(
2818 lambda: connect(uri),
2819 maxretry=attempt_count,
2820 period=1,
2821 exceptions=[
2822 linstor.errors.LinstorNetworkError,
2823 LinstorVolumeManagerError
2824 ]
2825 )
2827 @classmethod
2828 def _configure_volume_peer_slots(cls, lin, volume_name):
2829 result = lin.resource_dfn_modify(volume_name, {}, peer_slots=3)
2830 error_str = cls._get_error_str(result)
2831 if error_str:
2832 raise LinstorVolumeManagerError(
2833 'Could not configure volume peer slots of {}: {}'
2834 .format(volume_name, error_str)
2835 )
2837 @classmethod
2838 def _activate_device_path(cls, lin, node_name, volume_name):
2839 result = lin.resource_make_available(node_name, volume_name, diskful=False)
2840 if linstor.Linstor.all_api_responses_no_error(result):
2841 return
2842 errors = linstor.Linstor.filter_api_call_response_errors(result)
2843 if len(errors) == 1 and errors[0].is_error(
2844 linstor.consts.FAIL_EXISTS_RSC
2845 ):
2846 return
2848 raise LinstorVolumeManagerError(
2849 'Unable to activate device path of `{}` on node `{}`: {}'
2850 .format(volume_name, node_name, ', '.join(
2851 [str(x) for x in result]))
2852 )
2854 @classmethod
2855 def _request_database_path(cls, lin, activate=False):
2856 node_name = socket.gethostname()
2858 try:
2859 resource = next(filter(
2860 lambda resource: resource.node_name == node_name and
2861 resource.name == DATABASE_VOLUME_NAME,
2862 lin.resource_list_raise().resources
2863 ), None)
2864 except Exception as e:
2865 raise LinstorVolumeManagerError(
2866 'Unable to fetch database resource: {}'
2867 .format(e)
2868 )
2870 if not resource:
2871 if activate:
2872 cls._activate_device_path(
2873 lin, node_name, DATABASE_VOLUME_NAME
2874 )
2875 return cls._request_database_path(
2876 DATABASE_VOLUME_NAME, DATABASE_VOLUME_NAME
2877 )
2878 raise LinstorVolumeManagerError(
2879 'Empty dev path for `{}`, but definition "seems" to exist'
2880 .format(DATABASE_PATH)
2881 )
2882 # Contains a path of the /dev/drbd<id> form.
2883 return resource.volumes[0].device_path
2885 @classmethod
2886 def _create_database_volume(
2887 cls, lin, group_name, storage_pool_name, node_names, redundancy
2888 ):
2889 try:
2890 dfns = lin.resource_dfn_list_raise().resource_definitions
2891 except Exception as e:
2892 raise LinstorVolumeManagerError(
2893 'Unable to get definitions during database creation: {}'
2894 .format(e)
2895 )
2897 if dfns:
2898 raise LinstorVolumeManagerError(
2899 'Could not create volume `{}` from SR `{}`, '.format(
2900 DATABASE_VOLUME_NAME, group_name
2901 ) + 'LINSTOR volume list must be empty.'
2902 )
2904 # Workaround to use thin lvm. Without this line an error is returned:
2905 # "Not enough available nodes"
2906 # I don't understand why but this command protect against this bug.
2907 try:
2908 pools = lin.storage_pool_list_raise(
2909 filter_by_stor_pools=[storage_pool_name]
2910 )
2911 except Exception as e:
2912 raise LinstorVolumeManagerError(
2913 'Failed to get storage pool list before database creation: {}'
2914 .format(e)
2915 )
2917 # Ensure we have a correct list of storage pools.
2918 assert pools.storage_pools # We must have at least one storage pool!
2919 nodes_with_pool = list(map(lambda pool: pool.node_name, pools.storage_pools))
2920 for node_name in nodes_with_pool:
2921 assert node_name in node_names
2922 util.SMlog('Nodes with storage pool: {}'.format(nodes_with_pool))
2924 # Create the database definition.
2925 size = cls.round_up_volume_size(DATABASE_SIZE)
2926 cls._check_volume_creation_errors(lin.resource_group_spawn(
2927 rsc_grp_name=group_name,
2928 rsc_dfn_name=DATABASE_VOLUME_NAME,
2929 vlm_sizes=['{}B'.format(size)],
2930 definitions_only=True
2931 ), DATABASE_VOLUME_NAME, group_name)
2932 cls._configure_volume_peer_slots(lin, DATABASE_VOLUME_NAME)
2934 # Create real resources on the first nodes.
2935 resources = []
2937 diskful_nodes = []
2938 diskless_nodes = []
2939 for node_name in node_names:
2940 if node_name in nodes_with_pool:
2941 diskful_nodes.append(node_name)
2942 else:
2943 diskless_nodes.append(node_name)
2945 assert diskful_nodes
2946 for node_name in diskful_nodes[:redundancy]:
2947 util.SMlog('Create database diskful on {}'.format(node_name))
2948 resources.append(linstor.ResourceData(
2949 node_name=node_name,
2950 rsc_name=DATABASE_VOLUME_NAME,
2951 storage_pool=storage_pool_name
2952 ))
2953 # Create diskless resources on the remaining set.
2954 for node_name in diskful_nodes[redundancy:] + diskless_nodes:
2955 util.SMlog('Create database diskless on {}'.format(node_name))
2956 resources.append(linstor.ResourceData(
2957 node_name=node_name,
2958 rsc_name=DATABASE_VOLUME_NAME,
2959 diskless=True
2960 ))
2962 result = lin.resource_create(resources)
2963 error_str = cls._get_error_str(result)
2964 if error_str:
2965 raise LinstorVolumeManagerError(
2966 'Could not create database volume from SR `{}`: {}'.format(
2967 group_name, error_str
2968 )
2969 )
2971 # Create database and ensure path exists locally and
2972 # on replicated devices.
2973 current_device_path = cls._request_database_path(lin, activate=True)
2975 # Ensure diskless paths exist on other hosts. Otherwise PBDs can't be
2976 # plugged.
2977 for node_name in node_names:
2978 cls._activate_device_path(lin, node_name, DATABASE_VOLUME_NAME)
2980 # We use realpath here to get the /dev/drbd<id> path instead of
2981 # /dev/drbd/by-res/<resource_name>.
2982 expected_device_path = cls.build_device_path(DATABASE_VOLUME_NAME)
2983 util.wait_for_path(expected_device_path, 5)
2985 device_realpath = os.path.realpath(expected_device_path)
2986 if current_device_path != device_realpath:
2987 raise LinstorVolumeManagerError(
2988 'Invalid path, current={}, expected={} (realpath={})'
2989 .format(
2990 current_device_path,
2991 expected_device_path,
2992 device_realpath
2993 )
2994 )
2996 try:
2997 util.retry(
2998 lambda: util.pread2([DATABASE_MKFS, expected_device_path]),
2999 maxretry=5
3000 )
3001 except Exception as e:
3002 raise LinstorVolumeManagerError(
3003 'Failed to execute {} on database volume: {}'
3004 .format(DATABASE_MKFS, e)
3005 )
3007 return expected_device_path
3009 @classmethod
3010 def _destroy_database_volume(cls, lin, group_name):
3011 error_str = cls._get_error_str(
3012 lin.resource_dfn_delete(DATABASE_VOLUME_NAME)
3013 )
3014 if error_str:
3015 raise LinstorVolumeManagerError(
3016 'Could not destroy resource `{}` from SR `{}`: {}'
3017 .format(DATABASE_VOLUME_NAME, group_name, error_str)
3018 )
3020 @classmethod
3021 def _mount_database_volume(cls, volume_path, mount=True, force=False):
3022 try:
3023 # 1. Create a backup config folder.
3024 database_not_empty = bool(os.listdir(DATABASE_PATH))
3025 backup_path = cls._create_database_backup_path()
3027 # 2. Move the config in the mounted volume.
3028 if database_not_empty:
3029 cls._move_files(DATABASE_PATH, backup_path)
3031 cls._mount_volume(volume_path, DATABASE_PATH, mount)
3033 if database_not_empty:
3034 cls._move_files(backup_path, DATABASE_PATH, force)
3036 # 3. Remove useless backup directory.
3037 try:
3038 os.rmdir(backup_path)
3039 except Exception as e:
3040 raise LinstorVolumeManagerError(
3041 'Failed to remove backup path {} of LINSTOR config: {}'
3042 .format(backup_path, e)
3043 )
3044 except Exception as e:
3045 def force_exec(fn):
3046 try:
3047 fn()
3048 except Exception:
3049 pass
3051 if mount == cls.is_controller():
3052 force_exec(lambda: cls._move_files(
3053 DATABASE_PATH, backup_path
3054 ))
3055 force_exec(lambda: cls._mount_volume(
3056 volume_path, DATABASE_PATH, not mount
3057 ))
3059 if mount != cls.is_controller():
3060 force_exec(lambda: cls._move_files(
3061 backup_path, DATABASE_PATH
3062 ))
3064 force_exec(lambda: os.rmdir(backup_path))
3065 raise e
3067 @classmethod
3068 def _force_destroy_database_volume(cls, lin, group_name):
3069 try:
3070 cls._destroy_database_volume(lin, group_name)
3071 except Exception:
3072 pass
3074 @classmethod
3075 def _destroy_storage_pool(cls, lin, group_name, node_name):
3076 def destroy():
3077 result = lin.storage_pool_delete(node_name, group_name)
3078 errors = cls._filter_errors(result)
3079 if cls._check_errors(errors, [
3080 linstor.consts.FAIL_NOT_FOUND_STOR_POOL,
3081 linstor.consts.FAIL_NOT_FOUND_STOR_POOL_DFN
3082 ]):
3083 return
3085 if errors:
3086 raise LinstorVolumeManagerError(
3087 'Failed to destroy SP `{}` on node `{}`: {}'.format(
3088 group_name,
3089 node_name,
3090 cls._get_error_str(errors)
3091 )
3092 )
3094 # We must retry to avoid errors like:
3095 # "can not be deleted as volumes / snapshot-volumes are still using it"
3096 # after LINSTOR database volume destruction.
3097 return util.retry(destroy, maxretry=10)
3099 @classmethod
3100 def _create_resource_group(
3101 cls,
3102 lin,
3103 group_name,
3104 storage_pool_name,
3105 redundancy,
3106 destroy_old_group
3107 ):
3108 rg_creation_attempt = 0
3109 while True:
3110 result = lin.resource_group_create(
3111 name=group_name,
3112 place_count=redundancy,
3113 storage_pool=storage_pool_name,
3114 diskless_on_remaining=False
3115 )
3116 error_str = cls._get_error_str(result)
3117 if not error_str:
3118 break
3120 errors = cls._filter_errors(result)
3121 if destroy_old_group and cls._check_errors(errors, [
3122 linstor.consts.FAIL_EXISTS_RSC_GRP
3123 ]):
3124 rg_creation_attempt += 1
3125 if rg_creation_attempt < 2:
3126 try:
3127 cls._destroy_resource_group(lin, group_name)
3128 except Exception as e:
3129 error_str = 'Failed to destroy old and empty RG: {}'.format(e)
3130 else:
3131 continue
3133 raise LinstorVolumeManagerError(
3134 'Could not create RG `{}`: {}'.format(
3135 group_name, error_str
3136 )
3137 )
3139 result = lin.volume_group_create(group_name)
3140 error_str = cls._get_error_str(result)
3141 if error_str:
3142 raise LinstorVolumeManagerError(
3143 'Could not create VG `{}`: {}'.format(
3144 group_name, error_str
3145 )
3146 )
3148 @classmethod
3149 def _destroy_resource_group(cls, lin, group_name):
3150 def destroy():
3151 result = lin.resource_group_delete(group_name)
3152 errors = cls._filter_errors(result)
3153 if cls._check_errors(errors, [
3154 linstor.consts.FAIL_NOT_FOUND_RSC_GRP
3155 ]):
3156 return
3158 if errors:
3159 raise LinstorVolumeManagerError(
3160 'Failed to destroy RG `{}`: {}'
3161 .format(group_name, cls._get_error_str(errors))
3162 )
3164 return util.retry(destroy, maxretry=10)
3166 @classmethod
3167 def build_group_name(cls, base_name):
3168 # If thin provisioning is used we have a path like this:
3169 # `VG/LV`. "/" is not accepted by LINSTOR.
3170 return '{}{}'.format(cls.PREFIX_SR, base_name.replace('/', '_'))
3172 # Used to store important data in a HA context,
3173 # i.e. a replication count of 3.
3174 @classmethod
3175 def _build_ha_group_name(cls, base_name):
3176 return '{}{}'.format(cls.PREFIX_HA, base_name.replace('/', '_'))
3178 @classmethod
3179 def _check_volume_creation_errors(cls, result, volume_uuid, group_name):
3180 errors = cls._filter_errors(result)
3181 if cls._check_errors(errors, [
3182 linstor.consts.FAIL_EXISTS_RSC, linstor.consts.FAIL_EXISTS_RSC_DFN
3183 ]):
3184 raise LinstorVolumeManagerError(
3185 'Failed to create volume `{}` from SR `{}`, it already exists'
3186 .format(volume_uuid, group_name),
3187 LinstorVolumeManagerError.ERR_VOLUME_EXISTS
3188 )
3190 if cls._check_errors(errors, [linstor.consts.FAIL_NOT_FOUND_RSC_GRP]):
3191 raise LinstorVolumeManagerError(
3192 'Failed to create volume `{}` from SR `{}`, resource group doesn\'t exist'
3193 .format(volume_uuid, group_name),
3194 LinstorVolumeManagerError.ERR_GROUP_NOT_EXISTS
3195 )
3197 if errors:
3198 raise LinstorVolumeManagerError(
3199 'Failed to create volume `{}` from SR `{}`: {}'.format(
3200 volume_uuid,
3201 group_name,
3202 cls._get_error_str(errors)
3203 )
3204 )
3206 @classmethod
3207 def _move_files(cls, src_dir, dest_dir, force=False):
3208 def listdir(dir):
3209 ignored = ['lost+found']
3210 return [file for file in os.listdir(dir) if file not in ignored]
3212 try:
3213 if not force:
3214 files = listdir(dest_dir)
3215 if files:
3216 raise LinstorVolumeManagerError(
3217 'Cannot move files from {} to {} because destination '
3218 'contains: {}'.format(src_dir, dest_dir, files)
3219 )
3220 except LinstorVolumeManagerError:
3221 raise
3222 except Exception as e:
3223 raise LinstorVolumeManagerError(
3224 'Cannot list dir {}: {}'.format(dest_dir, e)
3225 )
3227 try:
3228 for file in listdir(src_dir):
3229 try:
3230 dest_file = os.path.join(dest_dir, file)
3231 if not force and os.path.exists(dest_file):
3232 raise LinstorVolumeManagerError(
3233 'Cannot move {} because it already exists in the '
3234 'destination'.format(file)
3235 )
3236 shutil.move(os.path.join(src_dir, file), dest_file)
3237 except LinstorVolumeManagerError:
3238 raise
3239 except Exception as e:
3240 raise LinstorVolumeManagerError(
3241 'Cannot move {}: {}'.format(file, e)
3242 )
3243 except Exception as e:
3244 if not force:
3245 try:
3246 cls._move_files(dest_dir, src_dir, force=True)
3247 except Exception:
3248 pass
3250 raise LinstorVolumeManagerError(
3251 'Failed to move files from {} to {}: {}'.format(
3252 src_dir, dest_dir, e
3253 )
3254 )
3256 @staticmethod
3257 def _create_database_backup_path():
3258 path = DATABASE_PATH + '-' + str(uuid.uuid4())
3259 try:
3260 os.mkdir(path)
3261 return path
3262 except Exception as e:
3263 raise LinstorVolumeManagerError(
3264 'Failed to create backup path {} of LINSTOR config: {}'
3265 .format(path, e)
3266 )
3268 @staticmethod
3269 def _get_filtered_properties(properties):
3270 return dict(properties.items())
3272 @staticmethod
3273 def _filter_errors(result):
3274 return [
3275 err for err in result
3276 if hasattr(err, 'is_error') and err.is_error()
3277 ]
3279 @staticmethod
3280 def _check_errors(result, codes):
3281 for err in result:
3282 for code in codes:
3283 if err.is_error(code):
3284 return True
3285 return False
3287 @classmethod
3288 def _controller_is_running(cls):
3289 return cls._service_is_running('linstor-controller')
3291 @classmethod
3292 def _start_controller(cls, start=True):
3293 return cls._start_service('linstor-controller', start)
3295 @staticmethod
3296 def _start_service(name, start=True):
3297 action = 'start' if start else 'stop'
3298 (ret, out, err) = util.doexec([
3299 'systemctl', action, name
3300 ])
3301 if ret != 0:
3302 raise LinstorVolumeManagerError(
3303 'Failed to {} {}: {} {}'
3304 .format(action, name, out, err)
3305 )
3307 @staticmethod
3308 def _service_is_running(name):
3309 (ret, out, err) = util.doexec([
3310 'systemctl', 'is-active', '--quiet', name
3311 ])
3312 return not ret
3314 @staticmethod
3315 def _is_mounted(mountpoint):
3316 (ret, out, err) = util.doexec(['mountpoint', '-q', mountpoint])
3317 return ret == 0
3319 @classmethod
3320 def _mount_volume(cls, volume_path, mountpoint, mount=True):
3321 if mount:
3322 try:
3323 util.pread(['mount', volume_path, mountpoint])
3324 except Exception as e:
3325 raise LinstorVolumeManagerError(
3326 'Failed to mount volume {} on {}: {}'
3327 .format(volume_path, mountpoint, e)
3328 )
3329 else:
3330 try:
3331 if cls._is_mounted(mountpoint):
3332 util.pread(['umount', mountpoint])
3333 except Exception as e:
3334 raise LinstorVolumeManagerError(
3335 'Failed to umount volume {} on {}: {}'
3336 .format(volume_path, mountpoint, e)
3337 )
3340# ==============================================================================
3342# Check if a path is a DRBD resource and log the process name/pid
3343# that opened it.
3344def log_drbd_openers(path):
3345 # Ignore if it's not a symlink to DRBD resource.
3346 if not path.startswith(DRBD_BY_RES_PATH):
3347 return
3349 # Compute resource name.
3350 res_name_end = path.find('/', len(DRBD_BY_RES_PATH))
3351 if res_name_end == -1:
3352 return
3353 res_name = path[len(DRBD_BY_RES_PATH):res_name_end]
3355 volume_end = path.rfind('/')
3356 if volume_end == res_name_end:
3357 return
3358 volume = path[volume_end + 1:]
3360 try:
3361 # Ensure path is a DRBD.
3362 drbd_path = os.path.realpath(path)
3363 stats = os.stat(drbd_path)
3364 if not stat.S_ISBLK(stats.st_mode) or os.major(stats.st_rdev) != 147:
3365 return
3367 # Find where the device is open.
3368 (ret, stdout, stderr) = util.doexec(['drbdadm', 'status', res_name])
3369 if ret != 0:
3370 util.SMlog('Failed to execute `drbdadm status` on `{}`: {}'.format(
3371 res_name, stderr
3372 ))
3373 return
3375 # Is it a local device?
3376 if stdout.startswith('{} role:Primary'.format(res_name)):
3377 util.SMlog(
3378 'DRBD resource `{}` is open on local host: {}'
3379 .format(path, get_local_volume_openers(res_name, volume))
3380 )
3381 return
3383 # Is it a remote device?
3384 util.SMlog(
3385 'DRBD resource `{}` is open on hosts: {}'
3386 .format(path, get_all_volume_openers(res_name, volume))
3387 )
3388 except Exception as e:
3389 util.SMlog(
3390 'Got exception while trying to determine where DRBD resource ' +
3391 '`{}` is open: {}'.format(path, e)
3392 )