diff --git a/PVE/API2/Storage/Config.pm b/PVE/API2/Storage/Config.pm index 4461ddd..59e0c82 100755 --- a/PVE/API2/Storage/Config.pm +++ b/PVE/API2/Storage/Config.pm @@ -4,8 +4,10 @@ use strict; use warnings; use PVE::SafeSyslog; +use PVE::Tools qw(extract_param); use PVE::Cluster qw(cfs_read_file cfs_write_file); use PVE::Storage; +use PVE::Storage::Plugin; use HTTP::Status qw(:constants); use Storable qw(dclone); use PVE::JSONSchema qw(get_standard_option); @@ -17,19 +19,18 @@ use base qw(PVE::RESTHandler); my @ctypes = qw(images vztmpl iso backup); -my $storage_type_enum = ['dir', 'nfs', 'lvm', 'iscsi']; +my $storage_type_enum = PVE::Storage::Plugin->lookup_types(); my $api_storage_config = sub { my ($cfg, $storeid) = @_; - my $scfg = dclone(PVE::Storage::storage_config ($cfg, $storeid)); + my $scfg = dclone(PVE::Storage::storage_config($cfg, $storeid)); $scfg->{storage} = $storeid; - delete $scfg->{priority}; $scfg->{digest} = $cfg->{digest}; - $scfg->{content} = PVE::Storage::content_hash_to_string($scfg->{content}); + $scfg->{content} = PVE::Storage::Plugin->encode_value($scfg->{type}, 'content', $scfg->{content}); if ($scfg->{nodes}) { - $scfg->{nodes} = join(',', keys(%{$scfg->{nodes}})); + $scfg->{nodes} = PVE::Storage::Plugin->encode_value($scfg->{type}, 'nodes', $scfg->{nodes}); } return $scfg; @@ -119,92 +120,27 @@ __PACKAGE__->register_method ({ permissions => { check => ['perm', '/storage', ['Datastore.Allocate']], }, - parameters => { - additionalProperties => 0, - properties => { - storage => get_standard_option('pve-storage-id'), - nodes => get_standard_option('pve-node-list', { optional => 1 }), - type => { - type => 'string', - enum => $storage_type_enum, - }, - path => { - type => 'string', format => 'pve-storage-path', - optional => 1, - }, - export => { - type => 'string', format => 'pve-storage-path', - optional => 1, - }, - server => { - type => 'string', format => 'pve-storage-server', - optional => 1, - }, - options => { - type => 'string', format => 'pve-storage-options', - optional => 1, - }, - target => { - type => 'string', - optional => 1, - }, - vgname => { - type => 'string', format => 'pve-storage-vgname', - optional => 1, - }, - base => { - type => 'string', format => 'pve-volume-id', - optional => 1, - }, - portal => { - type => 'string', format => 'pve-storage-portal-dns', - optional => 1, - }, - content => { - type => 'string', format => 'pve-storage-content-list', - optional => 1, - }, - disable => { - type => 'boolean', - optional => 1, - }, - maxfiles => { - type => 'integer', - optional => 1, - minimum => 0, - }, - shared => { - type => 'boolean', - optional => 1, - }, - 'format' => { - type => 'string', format => 'pve-storage-format', - optional => 1, - }, - }, - }, + parameters => PVE::Storage::Plugin->createSchema(), returns => { type => 'null' }, code => sub { my ($param) = @_; - my $type = $param->{type}; - delete $param->{type}; - - my $storeid = $param->{storage}; - delete $param->{storage}; + my $type = extract_param($param, 'type'); + my $storeid = extract_param($param, 'storage'); if ($param->{portal}) { $param->{portal} = PVE::Storage::resolv_portal($param->{portal}); } - my $opts = PVE::Storage::parse_options($storeid, $type, $param, 1); + my $plugin = PVE::Storage::Plugin->lookup($type); + my $opts = $plugin->check_config($storeid, $param, 1, 1); PVE::Storage::lock_storage_config( sub { my $cfg = cfs_read_file('storage.cfg'); - if (my $scfg = PVE::Storage::storage_config ($cfg, $storeid, 1)) { + if (my $scfg = PVE::Storage::storage_config($cfg, $storeid, 1)) { die "storage ID '$storeid' already defined\n"; } @@ -212,7 +148,7 @@ __PACKAGE__->register_method ({ if ($type eq 'lvm' && $opts->{base}) { - my ($baseid, $volname) = PVE::Storage::parse_volume_id ($opts->{base}); + my ($baseid, $volname) = PVE::Storage::parse_volume_id($opts->{base}); my $basecfg = PVE::Storage::storage_config ($cfg, $baseid, 1); die "base storage ID '$baseid' does not exist\n" if !$basecfg; @@ -222,11 +158,11 @@ __PACKAGE__->register_method ({ die "unsupported base type '$basecfg->{type}'"; } - my $path = PVE::Storage::path ($cfg, $opts->{base}); + my $path = PVE::Storage::path($cfg, $opts->{base}); PVE::Storage::activate_storage($cfg, $baseid); - PVE::Storage::lvm_create_volume_group ($path, $opts->{vgname}, $opts->{shared}); + PVE::Storage::LVMPlugin::lvm_create_volume_group($path, $opts->{vgname}, $opts->{shared}); } # try to activate if enabled on local node, @@ -239,6 +175,7 @@ __PACKAGE__->register_method ({ }, "create storage failed"); + return undef; }}); __PACKAGE__->register_method ({ @@ -250,64 +187,25 @@ __PACKAGE__->register_method ({ permissions => { check => ['perm', '/storage', ['Datastore.Allocate']], }, - parameters => { - additionalProperties => 0, - properties => { - storage => get_standard_option('pve-storage-id'), - nodes => get_standard_option('pve-node-list', { optional => 1 }), - content => { - type => 'string', format => 'pve-storage-content-list', - optional => 1, - }, - 'format' => { - type => 'string', format => 'pve-storage-format', - optional => 1, - }, - disable => { - type => 'boolean', - optional => 1, - }, - shared => { - type => 'boolean', - optional => 1, - }, - options => { - type => 'string', format => 'pve-storage-options', - optional => 1, - }, - maxfiles => { - type => 'integer', - optional => 1, - minimum => 0, - }, - digest => { - type => 'string', - description => 'Prevent changes if current configuration file has different SHA1 digest. This can be used to prevent concurrent modifications.', - maxLength => 40, - optional => 1, - } - }, - }, + parameters => PVE::Storage::Plugin->updateSchema(), returns => { type => 'null' }, code => sub { my ($param) = @_; - my $storeid = $param->{storage}; - delete($param->{storage}); - - my $digest = $param->{digest}; - delete($param->{digest}); + my $storeid = extract_param($param, 'storage'); + my $digest = extract_param($param, 'digest'); PVE::Storage::lock_storage_config( sub { my $cfg = cfs_read_file('storage.cfg'); - PVE::Storage::assert_if_modified ($cfg, $digest); + PVE::SectionConfig::assert_if_modified($cfg, $digest); - my $scfg = PVE::Storage::storage_config ($cfg, $storeid); + my $scfg = PVE::Storage::storage_config($cfg, $storeid); - my $opts = PVE::Storage::parse_options($storeid, $scfg->{type}, $param); + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + my $opts = $plugin->check_config($storeid, $param, 0, 1); foreach my $k (%$opts) { $scfg->{$k} = $opts->{$k}; @@ -339,18 +237,17 @@ __PACKAGE__->register_method ({ code => sub { my ($param) = @_; - my $storeid = $param->{storage}; - delete($param->{storage}); - + my $storeid = extract_param($param, 'storage'); + PVE::Storage::lock_storage_config( sub { my $cfg = cfs_read_file('storage.cfg'); die "can't remove storage - storage is used as base of another storage\n" - if PVE::Storage::storage_is_used ($cfg, $storeid); + if PVE::Storage::storage_is_used($cfg, $storeid); - delete ($cfg->{ids}->{$storeid}); + delete $cfg->{ids}->{$storeid}; cfs_write_file('storage.cfg', $cfg); diff --git a/PVE/Makefile b/PVE/Makefile index ae63b2c..83cb51e 100644 --- a/PVE/Makefile +++ b/PVE/Makefile @@ -3,4 +3,5 @@ .PHONY: install install: install -D -m 0644 Storage.pm ${DESTDIR}${PERLDIR}/PVE/Storage.pm + make -C Storage install make -C API2 install \ No newline at end of file diff --git a/PVE/Storage.pm b/PVE/Storage.pm index cd74a3a..1d5abbd 100755 --- a/PVE/Storage.pm +++ b/PVE/Storage.pm @@ -3,97 +3,39 @@ package PVE::Storage; use strict; use POSIX; use IO::Select; -use IO::Dir; use IO::File; -use Fcntl ':flock'; -use File::stat; use File::Basename; use File::Path; -use IPC::Open2; use Cwd 'abs_path'; -use Getopt::Long qw(GetOptionsFromArray); -use Socket; -use Digest::SHA; -use Net::Ping; -use PVE::Tools qw(run_command file_read_firstline trim dir_glob_regex dir_glob_foreach); -use PVE::Cluster qw(cfs_register_file cfs_read_file cfs_write_file cfs_lock_file); +use PVE::Tools qw(run_command file_read_firstline); +use PVE::Cluster qw(cfs_read_file cfs_lock_file); use PVE::Exception qw(raise_param_exc); use PVE::JSONSchema; use PVE::INotify; use PVE::RPCEnvironment; -my $ISCSIADM = '/usr/bin/iscsiadm'; +use PVE::Storage::Plugin; +use PVE::Storage::DirPlugin; +use PVE::Storage::LVMPlugin; +use PVE::Storage::NFSPlugin; +use PVE::Storage::ISCSIPlugin; + +# load and initialize all plugins +PVE::Storage::DirPlugin->register(); +PVE::Storage::LVMPlugin->register(); +PVE::Storage::NFSPlugin->register(); +PVE::Storage::ISCSIPlugin->register(); +PVE::Storage::Plugin->init(); + my $UDEVADM = '/sbin/udevadm'; -$ISCSIADM = undef if ! -X $ISCSIADM; - -# fixme: always_call_parser => 1 ?? -cfs_register_file ('storage.cfg', - \&parse_config, - \&write_config); - -# generic utility function +# PVE::Storage utility functions sub config { return cfs_read_file("storage.cfg"); } -sub check_iscsi_support { - my $noerr = shift; - - if (!$ISCSIADM) { - my $msg = "no iscsi support - please install open-iscsi"; - if ($noerr) { - warn "warning: $msg\n"; - return 0; - } - - die "error: $msg\n"; - } - - return 1; -} - -sub load_stable_scsi_paths { - - my $stable_paths = {}; - - my $stabledir = "/dev/disk/by-id"; - - if (my $dh = IO::Dir->new($stabledir)) { - while (defined(my $tmp = $dh->read)) { - # exclude filenames with part in name (same disk but partitions) - # use only filenames with scsi(with multipath i have the same device - # with dm-uuid-mpath , dm-name and scsi in name) - if($tmp !~ m/-part\d+$/ && $tmp =~ m/^scsi-/) { - my $path = "$stabledir/$tmp"; - my $bdevdest = readlink($path); - if ($bdevdest && $bdevdest =~ m|^../../([^/]+)|) { - $stable_paths->{$1}=$tmp; - } - } - } - $dh->close; - } - return $stable_paths; -} - -sub read_proc_mounts { - - local $/; # enable slurp mode - - my $data = ""; - if (my $fd = IO::File->new ("/proc/mounts", "r")) { - $data = <$fd>; - close ($fd); - } - - return $data; -} - -# PVE::Storage utility functions - sub lock_storage_config { my ($code, $errmsg) = @_; @@ -104,470 +46,6 @@ sub lock_storage_config { } } -my $confvars = { - path => 'path', - shared => 'bool', - disable => 'bool', - saferemove => 'bool', - format => 'format', - content => 'content', - server => 'server', - export => 'path', - vgname => 'vgname', - base => 'volume', - portal => 'portal', - target => 'target', - nodes => 'nodes', - options => 'options', - maxfiles => 'natural', -}; - -my $required_config = { - dir => ['path'], - nfs => ['path', 'server', 'export'], - lvm => ['vgname'], - iscsi => ['portal', 'target'], -}; - -my $fixed_config = { - dir => ['path'], - nfs => ['path', 'server', 'export'], - lvm => ['vgname', 'base'], - iscsi => ['portal', 'target'], -}; - -my $default_config = { - dir => { - path => 1, - nodes => 0, - shared => 0, - disable => 0, - maxfiles => 0, - content => [ { images => 1, rootdir => 1, vztmpl => 1, iso => 1, backup => 1, none => 1 }, - { images => 1, rootdir => 1 }], - format => [ { raw => 1, qcow2 => 1, vmdk => 1 } , 'raw' ], - }, - - nfs => { - path => 1, - nodes => 0, - disable => 0, - server => 1, - export => 1, - options => 0, - maxfiles => 0, - content => [ { images => 1, rootdir => 1, vztmpl => 1, iso => 1, backup => 1}, - { images => 1 }], - format => [ { raw => 1, qcow2 => 1, vmdk => 1 } , 'raw' ], - }, - - lvm => { - vgname => 1, - nodes => 0, - shared => 0, - disable => 0, - saferemove => 0, - content => [ {images => 1}, { images => 1 }], - base => 1, - }, - - iscsi => { - portal => 1, - target => 1, - nodes => 0, - disable => 0, - content => [ {images => 1, none => 1}, { images => 1 }], - }, -}; - -sub valid_content_types { - my ($stype) = @_; - - my $def = $default_config->{$stype}; - - return {} if !$def; - - return $def->{content}->[0]; -} - -sub content_hash_to_string { - my $hash = shift; - - my @cta; - foreach my $ct (keys %$hash) { - push @cta, $ct if $hash->{$ct}; - } - - return join(',', @cta); -} - -PVE::JSONSchema::register_format('pve-storage-path', \&verify_path); -sub verify_path { - my ($path, $noerr) = @_; - - # fixme: exclude more shell meta characters? - # we need absolute paths - if ($path !~ m|^/[^;\(\)]+|) { - return undef if $noerr; - die "value does not look like a valid absolute path\n"; - } - return $path; -} - -PVE::JSONSchema::register_format('pve-storage-server', \&verify_server); -sub verify_server { - my ($server, $noerr) = @_; - - # fixme: use better regex ? - # IP or DNS name - if ($server !~ m/^[[:alnum:]\-\.]+$/) { - return undef if $noerr; - die "value does not look like a valid server name or IP address\n"; - } - return $server; -} - -PVE::JSONSchema::register_format('pve-storage-portal', \&verify_portal); -sub verify_portal { - my ($portal, $noerr) = @_; - - # IP with optional port - if ($portal !~ m/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(:\d+)?$/) { - return undef if $noerr; - die "value does not look like a valid portal address\n"; - } - return $portal; -} - -PVE::JSONSchema::register_format('pve-storage-portal-dns', \&verify_portal_dns); -sub verify_portal_dns { - my ($portal, $noerr) = @_; - - # IP or DNS name with optional port - if ($portal !~ m/^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|[[:alnum:]\-\.]+)(:\d+)?$/) { - return undef if $noerr; - die "value does not look like a valid portal address\n"; - } - return $portal; -} - -PVE::JSONSchema::register_format('pve-storage-content', \&verify_content); -sub verify_content { - my ($ct, $noerr) = @_; - - my $valid_content = valid_content_types('dir'); # dir includes all types - - if (!$valid_content->{$ct}) { - return undef if $noerr; - die "invalid content type '$ct'\n"; - } - - return $ct; -} - -PVE::JSONSchema::register_format('pve-storage-format', \&verify_format); -sub verify_format { - my ($fmt, $noerr) = @_; - - if ($fmt !~ m/(raw|qcow2|vmdk)/) { - return undef if $noerr; - die "invalid format '$fmt'\n"; - } - - return $fmt; -} - -PVE::JSONSchema::register_format('pve-storage-options', \&verify_options); -sub verify_options { - my ($value, $noerr) = @_; - - # mount options (see man fstab) - if ($value !~ m/^\S+$/) { - return undef if $noerr; - die "invalid options '$value'\n"; - } - - return $value; -} - -sub check_type { - my ($stype, $ct, $key, $value, $storeid, $noerr) = @_; - - my $def = $default_config->{$stype}; - - if (!$def) { # should not happen - return undef if $noerr; - die "unknown storage type '$stype'\n"; - } - - if (!defined($def->{$key})) { - return undef if $noerr; - die "unexpected property\n"; - } - - if (!defined ($value)) { - return undef if $noerr; - die "got undefined value\n"; - } - - if ($value =~ m/[\n\r]/) { - return undef if $noerr; - die "property contains a line feed\n"; - } - - if ($ct eq 'bool') { - return 1 if ($value eq '1') || ($value =~ m/^(on|yes|true)$/i); - return 0 if ($value eq '0') || ($value =~ m/^(off|no|false)$/i); - return undef if $noerr; - die "type check ('boolean') failed - got '$value'\n"; - } elsif ($ct eq 'options') { - return verify_options($value, $noerr); - } elsif ($ct eq 'path') { - return verify_path($value, $noerr); - } elsif ($ct eq 'server') { - return verify_server($value, $noerr); - } elsif ($ct eq 'vgname') { - return parse_lvm_name ($value, $noerr); - } elsif ($ct eq 'portal') { - return verify_portal($value, $noerr); - } elsif ($ct eq 'natural') { - return int($value) if $value =~ m/^\d+$/; - return undef if $noerr; - die "type check ('natural') failed - got '$value'\n"; - } elsif ($ct eq 'nodes') { - my $res = {}; - - foreach my $node (PVE::Tools::split_list($value)) { - if (PVE::JSONSchema::pve_verify_node_name($node, $noerr)) { - $res->{$node} = 1; - } - } - - # no node restrictions for local storage - if ($storeid && $storeid eq 'local' && scalar(keys(%$res))) { - return undef if $noerr; - die "storage '$storeid' does not allow node restrictions\n"; - } - - return $res; - } elsif ($ct eq 'target') { - return $value; - } elsif ($ct eq 'string') { - return $value; - } elsif ($ct eq 'format') { - my $valid_formats = $def->{format}->[0]; - - if (!$valid_formats->{$value}) { - return undef if $noerr; - die "storage does not support format '$value'\n"; - } - - return $value; - - } elsif ($ct eq 'content') { - my $valid_content = $def->{content}->[0]; - - my $res = {}; - - foreach my $c (PVE::Tools::split_list($value)) { - if (!$valid_content->{$c}) { - return undef if $noerr; - die "storage does not support content type '$c'\n"; - } - $res->{$c} = 1; - } - - if ($res->{none} && scalar (keys %$res) > 1) { - return undef if $noerr; - die "unable to combine 'none' with other content types\n"; - } - - return $res; - } elsif ($ct eq 'volume') { - return $value if parse_volume_id ($value, $noerr); - } - - return undef if $noerr; - die "type check not implemented - internal error\n"; -} - -sub parse_config { - my ($filename, $raw) = @_; - - my $ids = {}; - - my $digest = Digest::SHA::sha1_hex(defined($raw) ? $raw : ''); - - my $pri = 0; - - while ($raw && $raw =~ s/^(.*?)(\n|$)//) { - my $line = $1; - - next if $line =~ m/^\#/; - next if $line =~ m/^\s*$/; - - if ($line =~ m/^(\S+):\s*(\S+)\s*$/) { - my $storeid = $2; - my $type = $1; - my $ignore = 0; - - if (!PVE::JSONSchema::parse_storage_id($storeid, 1)) { - $ignore = 1; - warn "ignoring storage '$storeid' - (illegal characters)\n"; - } elsif (!$default_config->{$type}) { - $ignore = 1; - warn "ignoring storage '$storeid' (unsupported type '$type')\n"; - } else { - $ids->{$storeid}->{type} = $type; - $ids->{$storeid}->{priority} = $pri++; - } - - while ($raw && $raw =~ s/^(.*?)(\n|$)//) { - $line = $1; - - next if $line =~ m/^\#/; - last if $line =~ m/^\s*$/; - - next if $ignore; # skip - - if ($line =~ m/^\s+(\S+)(\s+(.*\S))?\s*$/) { - my ($k, $v) = ($1, $3); - if (my $ct = $confvars->{$k}) { - $v = 1 if $ct eq 'bool' && !defined($v); - eval { - $ids->{$storeid}->{$k} = check_type ($type, $ct, $k, $v, $storeid); - }; - warn "storage '$storeid' - unable to parse value of '$k': $@" if $@; - } else { - warn "storage '$storeid' - unable to parse value of '$k'\n"; - } - - } else { - warn "storage '$storeid' - ignore config line: $line\n"; - } - } - } else { - warn "ignore config line: $line\n"; - } - } - - # make sure we have a reasonable 'local:' storage - # openvz expects things to be there - if (!$ids->{local} || $ids->{local}->{type} ne 'dir' || - $ids->{local}->{path} ne '/var/lib/vz') { - $ids->{local} = { - type => 'dir', - priority => $pri++, - path => '/var/lib/vz', - maxfiles => 0, - content => { images => 1, rootdir => 1, vztmpl => 1, iso => 1}, - }; - } - - # we always need this for OpenVZ - $ids->{local}->{content}->{rootdir} = 1; - $ids->{local}->{content}->{vztmpl} = 1; - delete ($ids->{local}->{disable}); - - # remove node restrictions for local storage - delete($ids->{local}->{nodes}); - - foreach my $storeid (keys %$ids) { - my $d = $ids->{$storeid}; - - my $req_keys = $required_config->{$d->{type}}; - foreach my $k (@$req_keys) { - if (!defined ($d->{$k})) { - warn "ignoring storage '$storeid' - missing value " . - "for required option '$k'\n"; - delete $ids->{$storeid}; - next; - } - } - - my $def = $default_config->{$d->{type}}; - - if ($def->{content}) { - $d->{content} = $def->{content}->[1] if !$d->{content}; - } - - if ($d->{type} eq 'iscsi' || $d->{type} eq 'nfs') { - $d->{shared} = 1; - } - } - - my $cfg = { ids => $ids, digest => $digest}; - - return $cfg; -} - -sub parse_options { - my ($storeid, $stype, $param, $create) = @_; - - my $settings = { type => $stype }; - - die "unknown storage type '$stype'\n" - if !$default_config->{$stype}; - - foreach my $opt (keys %$param) { - my $value = $param->{$opt}; - - my $ct = $confvars->{$opt}; - if (defined($value)) { - eval { - $settings->{$opt} = check_type ($stype, $ct, $opt, $value, $storeid); - }; - raise_param_exc({ $opt => $@ }) if $@; - } else { - raise_param_exc({ $opt => "got undefined value" }); - } - } - - if ($create) { - my $req_keys = $required_config->{$stype}; - foreach my $k (@$req_keys) { - - if ($stype eq 'nfs' && !$settings->{path}) { - $settings->{path} = "/mnt/pve/$storeid"; - } - - # check if we have a value for all required options - if (!defined ($settings->{$k})) { - raise_param_exc({ $k => "property is missing and it is not optional" }); - } - } - } else { - my $fixed_keys = $fixed_config->{$stype}; - foreach my $k (@$fixed_keys) { - - # only allow to change non-fixed values - - if (defined ($settings->{$k})) { - raise_param_exc({$k => "can't change value (fixed parameter)"}); - } - } - } - - return $settings; -} - -sub cluster_lock_storage { - my ($storeid, $shared, $timeout, $func, @param) = @_; - - my $res; - if (!$shared) { - my $lockid = "pve-storage-$storeid"; - my $lockdir = "/var/lock/pve-manager"; - mkdir $lockdir; - $res = PVE::Tools::lock_file("$lockdir/$lockid", $timeout, $func, @param); - die $@ if $@; - } else { - $res = PVE::Cluster::cfs_lock_storage($storeid, $timeout, $func, @param); - die $@ if $@; - } - return $res; -} - sub storage_config { my ($cfg, $storeid, $noerr) = @_; @@ -583,7 +61,7 @@ sub storage_config { sub storage_check_node { my ($cfg, $storeid, $node, $noerr) = @_; - my $scfg = storage_config ($cfg, $storeid); + my $scfg = storage_config($cfg, $storeid); if ($scfg->{nodes}) { $node = PVE::INotify::nodename() if !$node || ($node eq 'localhost'); @@ -599,7 +77,7 @@ sub storage_check_node { sub storage_check_enabled { my ($cfg, $storeid, $node, $noerr) = @_; - my $scfg = storage_config ($cfg, $storeid); + my $scfg = storage_config($cfg, $storeid); if ($scfg->{disable}) { die "storage '$storeid' is disabled\n" if !$noerr; @@ -612,320 +90,66 @@ sub storage_check_enabled { sub storage_ids { my ($cfg) = @_; - my $ids = $cfg->{ids}; - - my @sa = sort {$ids->{$a}->{priority} <=> $ids->{$b}->{priority}} keys %$ids; - - return @sa; + return keys %{$cfg->{ids}}; } -sub assert_if_modified { - my ($cfg, $digest) = @_; +sub file_size_info { + my ($filename, $timeout) = @_; - if ($digest && ($cfg->{digest} ne $digest)) { - die "detected modified storage configuration - try again\n"; - } -} - -sub sprint_config_line { - my ($k, $v) = @_; - - my $ct = $confvars->{$k}; - - if ($ct eq 'bool') { - return $v ? "\t$k\n" : ''; - } elsif ($ct eq 'nodes') { - my $nlist = join(',', keys(%$v)); - return $nlist ? "\tnodes $nlist\n" : ''; - } elsif ($ct eq 'content') { - my $clist = content_hash_to_string($v); - if ($clist) { - return "\t$k $clist\n"; - } else { - return "\t$k none\n"; - } - } else { - return "\t$k $v\n"; - } -} - -sub write_config { - my ($filename, $cfg) = @_; - - my $out = ''; - - my $ids = $cfg->{ids}; - - my $maxpri = 0; - foreach my $storeid (keys %$ids) { - my $pri = $ids->{$storeid}->{priority}; - $maxpri = $pri if $pri && $pri > $maxpri; - } - foreach my $storeid (keys %$ids) { - if (!defined ($ids->{$storeid}->{priority})) { - $ids->{$storeid}->{priority} = ++$maxpri; - } - } - - foreach my $storeid (sort {$ids->{$a}->{priority} <=> $ids->{$b}->{priority}} keys %$ids) { - my $scfg = $ids->{$storeid}; - my $type = $scfg->{type}; - my $def = $default_config->{$type}; - - die "unknown storage type '$type'\n" if !$def; - - my $data = "$type: $storeid\n"; - - $data .= "\tdisable\n" if $scfg->{disable}; - - my $done_hash = { disable => 1}; - foreach my $k (@{$required_config->{$type}}) { - $done_hash->{$k} = 1; - my $v = $ids->{$storeid}->{$k}; - die "storage '$storeid' - missing value for required option '$k'\n" - if !defined ($v); - $data .= sprint_config_line ($k, $v); - } - - foreach my $k (keys %$def) { - next if defined ($done_hash->{$k}); - my $v = $ids->{$storeid}->{$k}; - next if !defined($v); - $data .= sprint_config_line ($k, $v); - } - - $out .= "$data\n"; - } - - return $out; + return PVE::Storage::Plugin::file_size_info($filename, $timeout); } sub get_image_dir { my ($cfg, $storeid, $vmid) = @_; - my $path = $cfg->{ids}->{$storeid}->{path}; - return $vmid ? "$path/images/$vmid" : "$path/images"; + my $scfg = storage_config($cfg, $storeid); + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + + my $path = $plugin->get_subdir($scfg, 'images'); + + return $vmid ? "$path/$vmid" : $path; } sub get_private_dir { my ($cfg, $storeid, $vmid) = @_; - my $path = $cfg->{ids}->{$storeid}->{path}; - return $vmid ? "$path/private/$vmid" : "$path/private"; + my $scfg = storage_config($cfg, $storeid); + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + + my $path = $plugin->get_subdir($scfg, 'rootdir'); + + return $vmid ? "$path/$vmid" : $path; } sub get_iso_dir { my ($cfg, $storeid) = @_; - my $isodir = $cfg->{ids}->{$storeid}->{path}; - $isodir .= '/template/iso'; + my $scfg = storage_config($cfg, $storeid); + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); - return $isodir; + return $plugin->get_subdir($scfg, 'iso'); } sub get_vztmpl_dir { my ($cfg, $storeid) = @_; - my $tmpldir = $cfg->{ids}->{$storeid}->{path}; - $tmpldir .= '/template/cache'; + my $scfg = storage_config($cfg, $storeid); + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); - return $tmpldir; + return $plugin->get_subdir($scfg, 'vztmpl'); } sub get_backup_dir { my ($cfg, $storeid) = @_; - my $dir = $cfg->{ids}->{$storeid}->{path}; - $dir .= '/dump'; + my $scfg = storage_config($cfg, $storeid); + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); - return $dir; -} - -# iscsi utility functions - -sub iscsi_session_list { - - check_iscsi_support (); - - my $cmd = [$ISCSIADM, '--mode', 'session']; - - my $res = {}; - - run_command($cmd, outfunc => sub { - my $line = shift; - - if ($line =~ m/^tcp:\s+\[(\S+)\]\s+\S+\s+(\S+)\s*$/) { - my ($session, $target) = ($1, $2); - # there can be several sessions per target (multipath) - push @{$res->{$target}}, $session; - - } - }); - - return $res; -} - -sub iscsi_test_portal { - my ($portal) = @_; - - my ($server, $port) = split(':', $portal); - my $p = Net::Ping->new("tcp", 2); - $p->port_number($port || 3260); - return $p->ping($server); -} - -sub iscsi_discovery { - my ($portal) = @_; - - check_iscsi_support (); - - my $cmd = [$ISCSIADM, '--mode', 'discovery', '--type', 'sendtargets', - '--portal', $portal]; - - my $res = {}; - - return $res if !iscsi_test_portal($portal); # fixme: raise exception here? - - run_command($cmd, outfunc => sub { - my $line = shift; - - if ($line =~ m/^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d+)\,\S+\s+(\S+)\s*$/) { - my $portal = $1; - my $target = $2; - # one target can have more than one portal (multipath). - push @{$res->{$target}}, $portal; - } - }); - - return $res; -} - -sub iscsi_login { - my ($target, $portal_in) = @_; - - check_iscsi_support (); - - eval { iscsi_discovery ($portal_in); }; - warn $@ if $@; - - my $cmd = [$ISCSIADM, '--mode', 'node', '--targetname', $target, '--login']; - run_command($cmd); -} - -sub iscsi_logout { - my ($target, $portal) = @_; - - check_iscsi_support (); - - my $cmd = [$ISCSIADM, '--mode', 'node', '--targetname', $target, '--logout']; - run_command($cmd); -} - -my $rescan_filename = "/var/run/pve-iscsi-rescan.lock"; - -sub iscsi_session_rescan { - my $session_list = shift; - - check_iscsi_support (); - - my $rstat = stat ($rescan_filename); - - if (!$rstat) { - if (my $fh = IO::File->new ($rescan_filename, "a")) { - utime undef, undef, $fh; - close ($fh); - } - } else { - my $atime = $rstat->atime; - my $tdiff = time() - $atime; - # avoid frequent rescans - return if !($tdiff < 0 || $tdiff > 10); - utime undef, undef, $rescan_filename; - } - - foreach my $session (@$session_list) { - my $cmd = [$ISCSIADM, '--mode', 'session', '-r', $session, '-R']; - eval { run_command($cmd, outfunc => sub {}); }; - warn $@ if $@; - } -} - -sub iscsi_device_list { - - my $res = {}; - - my $dirname = '/sys/class/iscsi_session'; - - my $stable_paths = load_stable_scsi_paths(); - - dir_glob_foreach ($dirname, 'session(\d+)', sub { - my ($ent, $session) = @_; - - my $target = file_read_firstline ("$dirname/$ent/targetname"); - return if !$target; - - my (undef, $host) = dir_glob_regex ("$dirname/$ent/device", 'target(\d+):.*'); - return if !defined($host); - - dir_glob_foreach ("/sys/bus/scsi/devices", "$host:" . '(\d+):(\d+):(\d+)', sub { - my ($tmp, $channel, $id, $lun) = @_; - - my $type = file_read_firstline ("/sys/bus/scsi/devices/$tmp/type"); - return if !defined($type) || $type ne '0'; # list disks only - - my $bdev; - if (-d "/sys/bus/scsi/devices/$tmp/block") { # newer kernels - (undef, $bdev) = dir_glob_regex ("/sys/bus/scsi/devices/$tmp/block/", '([A-Za-z]\S*)'); - } else { - (undef, $bdev) = dir_glob_regex ("/sys/bus/scsi/devices/$tmp", 'block:(\S+)'); - } - return if !$bdev; - - #check multipath - if (-d "/sys/block/$bdev/holders") { - my $multipathdev = dir_glob_regex ("/sys/block/$bdev/holders", '[A-Za-z]\S*'); - $bdev = $multipathdev if $multipathdev; - } - - my $blockdev = $stable_paths->{$bdev}; - return if !$blockdev; - - my $size = file_read_firstline ("/sys/block/$bdev/size"); - return if !$size; - - my $volid = "$channel.$id.$lun.$blockdev"; - - $res->{$target}->{$volid} = { - 'format' => 'raw', - 'size' => int($size * 512), - 'vmid' => 0, # not assigned to any vm - 'channel' => int($channel), - 'id' => int($id), - 'lun' => int($lun), - }; - - #print "TEST: $target $session $host,$bus,$tg,$lun $blockdev\n"; - }); - - }); - - return $res; + return $plugin->get_subdir($scfg, 'backup'); } # library implementation -PVE::JSONSchema::register_format('pve-storage-vgname', \&parse_lvm_name); -sub parse_lvm_name { - my ($name, $noerr) = @_; - - if ($name !~ m/^[a-z][a-z0-9\-\_\.]*[a-z0-9]$/i) { - return undef if $noerr; - die "lvm name '$name' contains illegal characters\n"; - } - - return $name; -} - sub parse_vmid { my $vmid = shift; @@ -945,73 +169,18 @@ sub parse_volume_id { die "unable to parse volume ID '$volid'\n"; } -sub parse_name_dir { - my $name = shift; - - if ($name =~ m!^([^/\s]+\.(raw|qcow2|vmdk))$!) { - return ($1, $2); - } - - die "unable to parse volume filename '$name'\n"; -} - -sub parse_volname_dir { - my $volname = shift; - - if ($volname =~ m!^(\d+)/(\S+)$!) { - my ($vmid, $name) = ($1, $2); - parse_name_dir ($name); - return ('image', $name, $vmid); - } elsif ($volname =~ m!^iso/([^/]+\.[Ii][Ss][Oo])$!) { - return ('iso', $1); - } elsif ($volname =~ m!^vztmpl/([^/]+\.tar\.gz)$!) { - return ('vztmpl', $1); - } elsif ($volname =~ m!^rootdir/(\d+)$!) { - return ('rootdir', $1, $1); - } elsif ($volname =~ m!^backup/([^/]+(\.(tar|tar\.gz|tar\.lzo|tgz)))$!) { - my $fn = $1; - if ($fn =~ m/^vzdump-(openvz|qemu)-(\d+)-.+/) { - return ('backup', $fn, $2); - } - return ('backup', $fn); - } - die "unable to parse directory volume name '$volname'\n"; -} - -sub parse_volname_lvm { - my $volname = shift; - - parse_lvm_name ($volname); - - if ($volname =~ m/^(vm-(\d+)-\S+)$/) { - return ($1, $2); - } - - die "unable to parse lvm volume name '$volname'\n"; -} - -sub parse_volname_iscsi { - my $volname = shift; - - if ($volname =~ m!^\d+\.\d+\.\d+\.(\S+)$!) { - my $byid = $1; - return $byid; - } - - die "unable to parse iscsi volume name '$volname'\n"; -} - # try to map a filesystem path to a volume identifier sub path_to_volume_id { my ($cfg, $path) = @_; my $ids = $cfg->{ids}; - my ($sid, $volname) = parse_volume_id ($path, 1); + my ($sid, $volname) = parse_volume_id($path, 1); if ($sid) { - if ($ids->{$sid} && (my $type = $ids->{$sid}->{type})) { - if ($type eq 'dir' || $type eq 'nfs') { - my ($vtype, $name, $vmid) = parse_volname_dir ($volname); + if (my $scfg = $ids->{$sid}) { + if (my $path = $scfg->{path}) { + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + my ($vtype, $name, $vmid) = $plugin->parse_volname($volname); return ($vtype, $path); } } @@ -1023,19 +192,19 @@ sub path_to_volume_id { $path = abs_path($path) || $path; foreach my $sid (keys %$ids) { - my $type = $ids->{$sid}->{type}; - next if !($type eq 'dir' || $type eq 'nfs'); - - my $imagedir = get_image_dir($cfg, $sid); - my $isodir = get_iso_dir($cfg, $sid); - my $tmpldir = get_vztmpl_dir($cfg, $sid); - my $backupdir = get_backup_dir($cfg, $sid); - my $privatedir = get_private_dir($cfg, $sid); + my $scfg = $ids->{$sid}; + next if !$scfg->{path}; + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + my $imagedir = $plugin->get_subdir($scfg, 'images'); + my $isodir = $plugin->get_subdir($scfg, 'iso'); + my $tmpldir = $plugin->get_subdir($scfg, 'vztmpl'); + my $backupdir = $plugin->get_subdir($scfg, 'backup'); + my $privatedir = $plugin->get_subdir($scfg, 'rootdir'); if ($path =~ m!^$imagedir/(\d+)/([^/\s]+)$!) { my $vmid = $1; my $name = $2; - return ('image', "$sid:$vmid/$name"); + return ('images', "$sid:$vmid/$name"); } elsif ($path =~ m!^$isodir/([^/]+\.[Ii][Ss][Oo])$!) { my $name = $1; return ('iso', "$sid:iso/$name"); @@ -1058,55 +227,12 @@ sub path_to_volume_id { sub path { my ($cfg, $volid) = @_; - my ($storeid, $volname) = parse_volume_id ($volid); + my ($storeid, $volname) = parse_volume_id($volid); - my $scfg = storage_config ($cfg, $storeid); - - my $path; - my $owner; - my $vtype = 'image'; - - if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') { - my ($name, $vmid); - ($vtype, $name, $vmid) = parse_volname_dir ($volname); - $owner = $vmid; - - my $imagedir = get_image_dir($cfg, $storeid, $vmid); - my $isodir = get_iso_dir($cfg, $storeid); - my $tmpldir = get_vztmpl_dir($cfg, $storeid); - my $backupdir = get_backup_dir($cfg, $storeid); - my $privatedir = get_private_dir($cfg, $storeid); - - if ($vtype eq 'image') { - $path = "$imagedir/$name"; - } elsif ($vtype eq 'iso') { - $path = "$isodir/$name"; - } elsif ($vtype eq 'vztmpl') { - $path = "$tmpldir/$name"; - } elsif ($vtype eq 'rootdir') { - $path = "$privatedir/$name"; - } elsif ($vtype eq 'backup') { - $path = "$backupdir/$name"; - } else { - die "should not be reached"; - } - - } elsif ($scfg->{type} eq 'lvm') { - - my $vg = $scfg->{vgname}; - - my ($name, $vmid) = parse_volname_lvm ($volname); - $owner = $vmid; - - $path = "/dev/$vg/$name"; - - } elsif ($scfg->{type} eq 'iscsi') { - my $byid = parse_volname_iscsi ($volname); - $path = "/dev/disk/by-id/$byid"; - } else { - die "unknown storage type '$scfg->{type}'"; - } + my $scfg = storage_config($cfg, $storeid); + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + my ($path, $owner, $vtype) = $plugin->path($scfg, $volname); return wantarray ? ($path, $owner, $vtype) : $path; } @@ -1133,13 +259,16 @@ sub storage_migrate { local $ENV{RSYNC_RSH} = $ssh; - if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') { - if ($tcfg->{type} eq 'dir' || $tcfg->{type} eq 'nfs') { + # only implemented for file system based storage + if ($scfg->{path}) { + if ($tcfg->{path}) { - my $src = path ($cfg, $volid); - my $dst = path ($cfg, $target_volid); + my $src_plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + my $dst_plugin = PVE::Storage::Plugin->lookup($tcfg->{type}); + my $src = $src_plugin->path($scfg, $volid); + my $dst = $dst_plugin->path($tcfg, $target_volid); - my $dirname = dirname ($dst); + my $dirname = dirname($dst); if ($tcfg->{shared}) { # we can do a local copy @@ -1155,7 +284,7 @@ sub storage_migrate { # we use rsync with --sparse, so we can't use --inplace, # so we remove file on the target if it already exists to # save space - my ($size, $format) = file_size_info($src); + my ($size, $format) = PVE::Storage::Plugin::file_size_info($src); if ($format && ($format eq 'raw') && $size) { run_command(['/usr/bin/ssh', "root\@${target_host}", 'rm', '-f', $dst], @@ -1182,13 +311,9 @@ sub storage_migrate { } }); } - - } else { - die "$errstr - target type '$tcfg->{type}' not implemented\n"; } - } else { die "$errstr - source type '$scfg->{type}' not implemented\n"; } @@ -1205,314 +330,55 @@ sub vdisk_alloc { die "no VMID specified\n" if !$vmid; - $vmid = parse_vmid ($vmid); + $vmid = parse_vmid($vmid); - my $defformat = storage_default_format ($cfg, $storeid); + my $defformat = PVE::Storage::Plugin::default_format($scfg); $fmt = $defformat if !$fmt; - activate_storage ($cfg, $storeid); + activate_storage($cfg, $storeid); + + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); # lock shared storage - return cluster_lock_storage($storeid, $scfg->{shared}, undef, sub { - - if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') { - - my $imagedir = get_image_dir ($cfg, $storeid, $vmid); - - mkpath $imagedir; - - if (!$name) { - - for (my $i = 1; $i < 100; $i++) { - my @gr = <$imagedir/vm-$vmid-disk-$i.*>; - if (!scalar(@gr)) { - $name = "vm-$vmid-disk-$i.$fmt"; - last; - } - } - } - - die "unable to allocate an image name for VM $vmid in storage '$storeid'\n" - if !$name; - - my (undef, $tmpfmt) = parse_name_dir ($name); - - die "illegal name '$name' - wrong extension for format ('$tmpfmt != '$fmt')\n" - if $tmpfmt ne $fmt; - - my $path = "$imagedir/$name"; - - die "disk image '$path' already exists\n" if -e $path; - - run_command("/usr/bin/qemu-img create -f $fmt '$path' ${size}K", - errmsg => "unable to create image"); - - return "$storeid:$vmid/$name"; - - } elsif ($scfg->{type} eq 'lvm') { - - die "unsupported format '$fmt'" if $fmt ne 'raw'; - - die "illegal name '$name' - sould be 'vm-$vmid-*'\n" - if $name && $name !~ m/^vm-$vmid-/; - - my $vgs = lvm_vgs (); - - my $vg = $scfg->{vgname}; - - die "no such volume gruoup '$vg'\n" if !defined ($vgs->{$vg}); - - my $free = int ($vgs->{$vg}->{free}); - - die "not enough free space ($free < $size)\n" if $free < $size; - - if (!$name) { - my $lvs = lvm_lvs ($vg); - - for (my $i = 1; $i < 100; $i++) { - my $tn = "vm-$vmid-disk-$i"; - if (!defined ($lvs->{$vg}->{$tn})) { - $name = $tn; - last; - } - } - } - - die "unable to allocate an image name for VM $vmid in storage '$storeid'\n" - if !$name; - - my $cmd = ['/sbin/lvcreate', '-aly', '--addtag', "pve-vm-$vmid", '--size', "${size}k", '--name', $name, $vg]; - - run_command($cmd, errmsg => "lvcreate '$vg/pve-vm-$vmid' error"); - - return "$storeid:$name"; - - } elsif ($scfg->{type} eq 'iscsi') { - die "can't allocate space in iscsi storage\n"; - } else { - die "unknown storage type '$scfg->{type}'"; - } + return $plugin->cluster_lock_storage($storeid, $scfg->{shared}, undef, sub { + my $volname = $plugin->alloc_image($storeid, $scfg, $vmid, $fmt, $name, $size); + return "$storeid:$volname"; }); } sub vdisk_free { my ($cfg, $volid) = @_; - my ($storeid, $volname) = parse_volume_id ($volid); + my ($storeid, $volname) = parse_volume_id($volid); - my $scfg = storage_config ($cfg, $storeid); + my $scfg = storage_config($cfg, $storeid); - activate_storage ($cfg, $storeid); + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + + activate_storage($cfg, $storeid); - # we need to zero out LVM data for security reasons - # and to allow thin provisioning - - my $vg; + my $cleanup_worker; # lock shared storage - cluster_lock_storage($storeid, $scfg->{shared}, undef, sub { - - if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') { - my $path = path ($cfg, $volid); - - if (! -f $path) { - warn "disk image '$path' does not exists\n"; - } else { - unlink $path; - } - } elsif ($scfg->{type} eq 'lvm') { - - if ($scfg->{saferemove}) { - # avoid long running task, so we only rename here - $vg = $scfg->{vgname}; - my $cmd = ['/sbin/lvrename', $vg, $volname, "del-$volname"]; - run_command($cmd, errmsg => "lvrename '$vg/$volname' error"); - } else { - my $tmpvg = $scfg->{vgname}; - my $cmd = ['/sbin/lvremove', '-f', "$tmpvg/$volname"]; - run_command($cmd, errmsg => "lvremove '$tmpvg/$volname' error"); - } - - } elsif ($scfg->{type} eq 'iscsi') { - die "can't free space in iscsi storage\n"; - } else { - die "unknown storage type '$scfg->{type}'"; - } + $plugin->cluster_lock_storage($storeid, $scfg->{shared}, undef, sub { + my $cleanup_worker = $plugin->free_image($storeid, $scfg, $volname); }); - return if !$vg; - - my $zero_out_worker = sub { - print "zero-out data on image $volname\n"; - my $cmd = ['dd', "if=/dev/zero", "of=/dev/$vg/del-$volname", "bs=1M"]; - eval { run_command($cmd, errmsg => "zero out failed"); }; - warn $@ if $@; - - cluster_lock_storage($storeid, $scfg->{shared}, undef, sub { - my $cmd = ['/sbin/lvremove', '-f', "$vg/del-$volname"]; - run_command($cmd, errmsg => "lvremove '$vg/del-$volname' error"); - }); - print "successfully removed volume $volname\n"; - }; + return if !$cleanup_worker; my $rpcenv = PVE::RPCEnvironment::get(); my $authuser = $rpcenv->get_user(); - $rpcenv->fork_worker('imgdel', undef, $authuser, $zero_out_worker); -} - -# lvm utility functions - -sub lvm_pv_info { - my ($device) = @_; - - die "no device specified" if !$device; - - my $has_label = 0; - - my $cmd = ['/usr/bin/file', '-L', '-s', $device]; - run_command($cmd, outfunc => sub { - my $line = shift; - $has_label = 1 if $line =~ m/LVM2/; - }); - - return undef if !$has_label; - - $cmd = ['/sbin/pvs', '--separator', ':', '--noheadings', '--units', 'k', - '--unbuffered', '--nosuffix', '--options', - 'pv_name,pv_size,vg_name,pv_uuid', $device]; - - my $pvinfo; - run_command($cmd, outfunc => sub { - my $line = shift; - - $line = trim($line); - - my ($pvname, $size, $vgname, $uuid) = split (':', $line); - - die "found multiple pvs entries for device '$device'\n" - if $pvinfo; - - $pvinfo = { - pvname => $pvname, - size => $size, - vgname => $vgname, - uuid => $uuid, - }; - }); - - return $pvinfo; -} - -sub clear_first_sector { - my ($dev) = shift; - - if (my $fh = IO::File->new ($dev, "w")) { - my $buf = 0 x 512; - syswrite $fh, $buf; - $fh->close(); - } -} - -sub lvm_create_volume_group { - my ($device, $vgname, $shared) = @_; - - my $res = lvm_pv_info ($device); - - if ($res->{vgname}) { - return if $res->{vgname} eq $vgname; # already created - die "device '$device' is already used by volume group '$res->{vgname}'\n"; - } - - clear_first_sector ($device); # else pvcreate fails - - # we use --metadatasize 250k, which reseults in "pe_start = 512" - # so pe_start is aligned on a 128k boundary (advantage for SSDs) - my $cmd = ['/sbin/pvcreate', '--metadatasize', '250k', $device]; - - run_command($cmd, errmsg => "pvcreate '$device' error"); - - $cmd = ['/sbin/vgcreate', $vgname, $device]; - # push @$cmd, '-c', 'y' if $shared; # we do not use this yet - - run_command($cmd, errmsg => "vgcreate $vgname $device error"); -} - -sub lvm_vgs { - - my $cmd = ['/sbin/vgs', '--separator', ':', '--noheadings', '--units', 'b', - '--unbuffered', '--nosuffix', '--options', - 'vg_name,vg_size,vg_free']; - - my $vgs = {}; - eval { - run_command($cmd, outfunc => sub { - my $line = shift; - - $line = trim($line); - - my ($name, $size, $free) = split (':', $line); - - $vgs->{$name} = { size => int ($size), free => int ($free) }; - }); - }; - my $err = $@; - - # just warn (vgs return error code 5 if clvmd does not run) - # but output is still OK (list without clustered VGs) - warn $err if $err; - - return $vgs; -} - -sub lvm_lvs { - my ($vgname) = @_; - - my $cmd = ['/sbin/lvs', '--separator', ':', '--noheadings', '--units', 'b', - '--unbuffered', '--nosuffix', '--options', - 'vg_name,lv_name,lv_size,uuid,tags']; - - push @$cmd, $vgname if $vgname; - - my $lvs = {}; - run_command($cmd, outfunc => sub { - my $line = shift; - - $line = trim($line); - - my ($vg, $name, $size, $uuid, $tags) = split (':', $line); - - return if $name !~ m/^vm-(\d+)-/; - my $nid = $1; - - my $owner; - foreach my $tag (split (/,/, $tags)) { - if ($tag =~ m/^pve-vm-(\d+)$/) { - $owner = $1; - last; - } - } - - if ($owner) { - if ($owner ne $nid) { - warn "owner mismatch name = $name, owner = $owner\n"; - } - - $lvs->{$vg}->{$name} = { format => 'raw', size => $size, - uuid => $uuid, tags => $tags, - vmid => $owner }; - } - }); - - return $lvs; + $rpcenv->fork_worker('imgdel', undef, $authuser, $cleanup_worker); } #list iso or openvz template ($tt = ) sub template_list { my ($cfg, $storeid, $tt) = @_; - die "unknown template type '$tt'\n" if !($tt eq 'iso' || $tt eq 'vztmpl' || $tt eq 'backup'); + die "unknown template type '$tt'\n" + if !($tt eq 'iso' || $tt eq 'vztmpl' || $tt eq 'backup'); my $ids = $cfg->{ids}; @@ -1534,20 +400,12 @@ sub template_list { next if $tt eq 'vztmpl' && !$scfg->{content}->{vztmpl}; next if $tt eq 'backup' && !$scfg->{content}->{backup}; - activate_storage ($cfg, $sid); + activate_storage($cfg, $sid); - if ($type eq 'dir' || $type eq 'nfs') { + if ($scfg->{path}) { + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); - my $path; - if ($tt eq 'iso') { - $path = get_iso_dir($cfg, $sid); - } elsif ($tt eq 'vztmpl') { - $path = get_vztmpl_dir($cfg, $sid); - } elsif ($tt eq 'backup') { - $path = get_backup_dir($cfg, $sid); - } else { - die "unknown template type '$tt'\n"; - } + my $path = $plugin->get_subdir($scfg, $tt); foreach my $fn (<$path/*>) { @@ -1582,40 +440,6 @@ sub template_list { return $res; } -sub file_size_info { - my ($filename, $timeout) = @_; - - my $cmd = ['/usr/bin/qemu-img', 'info', $filename]; - - my $format; - my $size = 0; - my $used = 0; - - eval { - run_command($cmd, timeout => $timeout, outfunc => sub { - my $line = shift; - - if ($line =~ m/^file format:\s+(\S+)\s*$/) { - $format = $1; - } elsif ($line =~ m/^virtual size:\s\S+\s+\((\d+)\s+bytes\)$/) { - $size = int($1); - } elsif ($line =~ m/^disk size:\s+(\d+(.\d+)?)([KMGT])\s*$/) { - $used = $1; - my $u = $3; - - $used *= 1024 if $u eq 'K'; - $used *= (1024*1024) if $u eq 'M'; - $used *= (1024*1024*1024) if $u eq 'G'; - $used *= (1024*1024*1024*1024) if $u eq 'T'; - - $used = int($used); - } - }); - }; - - return wantarray ? ($size, $format, $used) : $size; -} - sub vdisk_list { my ($cfg, $storeid, $vmid, $vollist) = @_; @@ -1627,172 +451,48 @@ sub vdisk_list { # prepare/activate/refresh all storages - my $stypes = {}; - my $storage_list = []; if ($vollist) { foreach my $volid (@$vollist) { - my ($sid, undef) = parse_volume_id ($volid); - next if !defined ($ids->{$sid}); + my ($sid, undef) = parse_volume_id($volid); + next if !defined($ids->{$sid}); next if !storage_check_enabled($cfg, $sid, undef, 1); push @$storage_list, $sid; - $stypes->{$ids->{$sid}->{type}} = 1; } } else { foreach my $sid (keys %$ids) { next if $storeid && $storeid ne $sid; next if !storage_check_enabled($cfg, $sid, undef, 1); push @$storage_list, $sid; - $stypes->{$ids->{$sid}->{type}} = 1; } } - activate_storage_list ($cfg, $storage_list); + my $cache = {}; - my $lvs = $stypes->{lvm} ? lvm_lvs () : {}; - - my $iscsi_devices = iscsi_device_list() if $stypes->{iscsi}; - - # query the storage + activate_storage_list($cfg, $storage_list, $cache); foreach my $sid (keys %$ids) { - if ($storeid) { - next if $storeid ne $sid; - next if !storage_check_enabled($cfg, $sid, undef, 1); - } + next if $storeid && $storeid ne $sid; + next if !storage_check_enabled($cfg, $sid, undef, 1); + my $scfg = $ids->{$sid}; - my $type = $scfg->{type}; - - if ($type eq 'dir' || $type eq 'nfs') { - - my $path = $scfg->{path}; - - my $fmts = join ('|', keys %{$default_config->{$type}->{format}->[0]}); - - foreach my $fn (<$path/images/[0-9][0-9]*/*>) { - - next if $fn !~ m!^(/.+/images/(\d+)/([^/]+\.($fmts)))$!; - $fn = $1; # untaint - - my $owner = $2; - my $name = $3; - my $volid = "$sid:$owner/$name"; - - if ($vollist) { - my $found = grep { $_ eq $volid } @$vollist; - next if !$found; - } else { - next if defined ($vmid) && ($owner ne $vmid); - } - - my ($size, $format, $used) = file_size_info ($fn); - - if ($format && $size) { - push @{$res->{$sid}}, { - volid => $volid, format => $format, - size => $size, vmid => $owner, used => $used }; - } - - } - - } elsif ($type eq 'lvm') { - - my $vgname = $scfg->{vgname}; - - if (my $dat = $lvs->{$vgname}) { - - foreach my $volname (keys %$dat) { - - my $owner = $dat->{$volname}->{vmid}; - - my $volid = "$sid:$volname"; - - if ($vollist) { - my $found = grep { $_ eq $volid } @$vollist; - next if !$found; - } else { - next if defined ($vmid) && ($owner ne $vmid); - } - - my $info = $dat->{$volname}; - $info->{volid} = $volid; - - push @{$res->{$sid}}, $info; - } - } - - } elsif ($type eq 'iscsi') { - - # we have no owner for iscsi devices - - my $target = $scfg->{target}; - - if (my $dat = $iscsi_devices->{$target}) { - - foreach my $volname (keys %$dat) { - - my $volid = "$sid:$volname"; - - if ($vollist) { - my $found = grep { $_ eq $volid } @$vollist; - next if !$found; - } else { - next if !($storeid && ($storeid eq $sid)); - } - - my $info = $dat->{$volname}; - $info->{volid} = $volid; - - push @{$res->{$sid}}, $info; - } - } - - } else { - die "implement me"; - } - + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + $res->{$sid} = $plugin->list_images($sid, $scfg, $vmid, $vollist, $cache); @{$res->{$sid}} = sort {lc($a->{volid}) cmp lc ($b->{volid}) } @{$res->{$sid}} if $res->{$sid}; } return $res; } -sub nfs_is_mounted { - my ($server, $export, $mountpoint, $mountdata) = @_; - - my $source = "$server:$export"; - - $mountdata = read_proc_mounts() if !$mountdata; - - if ($mountdata =~ m|^$source/?\s$mountpoint\snfs|m) { - return $mountpoint; - } - - return undef; -} - -sub nfs_mount { - my ($server, $export, $mountpoint, $options) = @_; - - my $source = "$server:$export"; - - my $cmd = ['/bin/mount', '-t', 'nfs', $source, $mountpoint]; - if ($options) { - push @$cmd, '-o', $options; - } - - run_command($cmd, errmsg => "mount error"); -} - sub uevent_seqnum { my $filename = "/sys/kernel/uevent_seqnum"; my $seqnum = 0; - if (my $fh = IO::File->new ($filename, "r")) { + if (my $fh = IO::File->new($filename, "r")) { my $line = <$fh>; if ($line =~ m/^(\d+)$/) { - $seqnum = int ($1); + $seqnum = int($1); } close ($fh); } @@ -1800,144 +500,62 @@ sub uevent_seqnum { } sub __activate_storage_full { - my ($cfg, $storeid, $session) = @_; + my ($cfg, $storeid, $cache) = @_; my $scfg = storage_check_enabled($cfg, $storeid); - return if $session->{activated}->{$storeid}; + return if $cache->{activated}->{$storeid}; - if (!$session->{mountdata}) { - $session->{mountdata} = read_proc_mounts(); + $cache->{uevent_seqnum} = uevent_seqnum() if !$cache->{uevent_seqnum}; + + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + + if ($scfg->{base}) { + my ($baseid, undef) = parse_volume_id ($scfg->{base}); + __activate_storage_full ($cfg, $baseid, $cache); } - if (!$session->{uevent_seqnum}) { - $session->{uevent_seqnum} = uevent_seqnum (); - } - - my $mountdata = $session->{mountdata}; - - my $type = $scfg->{type}; - - if ($type eq 'dir' || $type eq 'nfs') { - - my $path = $scfg->{path}; - - if ($type eq 'nfs') { - my $server = $scfg->{server}; - my $export = $scfg->{export}; - - if (!nfs_is_mounted ($server, $export, $path, $mountdata)) { - - # NOTE: only call mkpath when not mounted (avoid hang - # when NFS server is offline - - mkpath $path; - - die "unable to activate storage '$storeid' - " . - "directory '$path' does not exist\n" if ! -d $path; - - nfs_mount ($server, $export, $path, $scfg->{options}); - } - - } else { - - mkpath $path; - - die "unable to activate storage '$storeid' - " . - "directory '$path' does not exist\n" if ! -d $path; - } - - my $imagedir = get_image_dir($cfg, $storeid); - my $isodir = get_iso_dir($cfg, $storeid); - my $tmpldir = get_vztmpl_dir($cfg, $storeid); - my $backupdir = get_backup_dir($cfg, $storeid); - my $privatedir = get_private_dir($cfg, $storeid); - - if (defined($scfg->{content})) { - mkpath $imagedir if $scfg->{content}->{images} && - $imagedir ne $path; - mkpath $isodir if $scfg->{content}->{iso} && - $isodir ne $path; - mkpath $tmpldir if $scfg->{content}->{vztmpl} && - $tmpldir ne $path; - mkpath $privatedir if $scfg->{content}->{rootdir} && - $privatedir ne $path; - mkpath $backupdir if $scfg->{content}->{backup} && - $backupdir ne $path; - } - - } elsif ($type eq 'lvm') { - - if ($scfg->{base}) { - my ($baseid, undef) = parse_volume_id ($scfg->{base}); - __activate_storage_full ($cfg, $baseid, $session); - } - - if (!$session->{vgs}) { - $session->{vgs} = lvm_vgs(); - } - - # In LVM2, vgscans take place automatically; - # this is just to be sure - if ($session->{vgs} && !$session->{vgscaned} && - !$session->{vgs}->{$scfg->{vgname}}) { - $session->{vgscaned} = 1; - my $cmd = ['/sbin/vgscan', '--ignorelockingfailure', '--mknodes']; - eval { run_command($cmd, outfunc => sub {}); }; - warn $@ if $@; - } - - # we do not acticate any volumes here ('vgchange -aly') - # instead, volumes are activate individually later - - } elsif ($type eq 'iscsi') { - - return if !check_iscsi_support(1); - - $session->{iscsi_sessions} = iscsi_session_list() - if !$session->{iscsi_sessions}; - - my $iscsi_sess = $session->{iscsi_sessions}->{$scfg->{target}}; - if (!defined ($iscsi_sess)) { - eval { iscsi_login ($scfg->{target}, $scfg->{portal}); }; - warn $@ if $@; - } else { - # make sure we get all devices - iscsi_session_rescan ($iscsi_sess); - } - - } else { - die "implement me"; - } + $plugin->activate_storage($storeid, $scfg, $cache); my $newseq = uevent_seqnum (); # only call udevsettle if there are events - if ($newseq > $session->{uevent_seqnum}) { + if ($newseq > $cache->{uevent_seqnum}) { my $timeout = 30; system ("$UDEVADM settle --timeout=$timeout"); # ignore errors - $session->{uevent_seqnum} = $newseq; + $cache->{uevent_seqnum} = $newseq; } - $session->{activated}->{$storeid} = 1; + $cache->{activated}->{$storeid} = 1; } sub activate_storage_list { - my ($cfg, $storeid_list, $session) = @_; + my ($cfg, $storeid_list, $cache) = @_; - $session = {} if !$session; + $cache = {} if !$cache; foreach my $storeid (@$storeid_list) { - __activate_storage_full ($cfg, $storeid, $session); + __activate_storage_full ($cfg, $storeid, $cache); } } sub activate_storage { my ($cfg, $storeid) = @_; - my $session = {}; + my $cache = {}; - __activate_storage_full ($cfg, $storeid, $session); + __activate_storage_full ($cfg, $storeid, $cache); +} + + +sub deactivate_storage { + my ($cfg, $storeid) = @_; + + my $scfg = storage_config ($cfg, $storeid); + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + + my $cache = {}; + $plugin->deactivate_storage($storeid, $scfg, $cache); } sub activate_volumes { @@ -1945,34 +563,21 @@ sub activate_volumes { return if !($vollist && scalar(@$vollist)); - my $lvm_activate_mode = $exclusive ? 'ey' : 'ly'; - my $storagehash = {}; foreach my $volid (@$vollist) { - my ($storeid, undef) = parse_volume_id ($volid); + my ($storeid, undef) = parse_volume_id($volid); $storagehash->{$storeid} = 1; } - activate_storage_list ($cfg, [keys %$storagehash]); + my $cache = {}; + + activate_storage_list($cfg, [keys %$storagehash], $cache); foreach my $volid (@$vollist) { my ($storeid, $volname) = parse_volume_id ($volid); - - my $scfg = storage_config ($cfg, $storeid); - - my $path = path ($cfg, $volid); - - if ($scfg->{type} eq 'lvm') { - my $cmd = ['/sbin/lvchange', "-a$lvm_activate_mode", $path]; - run_command($cmd, errmsg => "can't activate LV '$volid'"); - } - - # check is volume exists - if ($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs') { - die "volume '$volid' does not exist\n" if ! -e $path; - } else { - die "volume '$volid' does not exist\n" if ! -b $path; - } + my $scfg = storage_config($cfg, $storeid); + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + $plugin->activate_volume($storeid, $scfg, $volname, $exclusive, $cache); } } @@ -1981,22 +586,21 @@ sub deactivate_volumes { return if !($vollist && scalar(@$vollist)); + my $cache = {}; + my @errlist = (); foreach my $volid (@$vollist) { - my ($storeid, $volname) = parse_volume_id ($volid); + my ($storeid, $volname) = parse_volume_id($volid); - my $scfg = storage_config ($cfg, $storeid); - - if ($scfg->{type} eq 'lvm') { - my $path = path ($cfg, $volid); - next if ! -b $path; - - my $cmd = ['/sbin/lvchange', '-aln', $path]; - eval { run_command($cmd, errmsg => "can't deactivate LV '$volid'"); }; - if (my $err = $@) { - warn $err; - push @errlist, $volid; - } + my $scfg = storage_config($cfg, $storeid); + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + + eval { + $plugin->deactivate_volume($storeid, $scfg, $volname, $cache); + }; + if (my $err = $@) { + warn $err; + push @errlist, $volid; } } @@ -2004,51 +608,12 @@ sub deactivate_volumes { if scalar(@errlist); } -sub deactivate_storage { - my ($cfg, $storeid) = @_; - - my $iscsi_sessions; - - my $scfg = storage_config ($cfg, $storeid); - - my $type = $scfg->{type}; - - if ($type eq 'dir') { - # nothing to do - } elsif ($type eq 'nfs') { - my $mountdata = read_proc_mounts(); - my $server = $scfg->{server}; - my $export = $scfg->{export}; - my $path = $scfg->{path}; - - my $cmd = ['/bin/umount', $path]; - - run_command($cmd, errmsg => 'umount error') - if nfs_is_mounted ($server, $export, $path, $mountdata); - - } elsif ($type eq 'lvm') { - my $cmd = ['/sbin/vgchange', '-aln', $scfg->{vgname}]; - run_command($cmd, errmsg => "can't deactivate VG '$scfg->{vgname}'"); - } elsif ($type eq 'iscsi') { - my $portal = $scfg->{portal}; - my $target = $scfg->{target}; - - my $iscsi_sessions = iscsi_session_list(); - iscsi_logout ($target, $portal) - if defined ($iscsi_sessions->{$target}); - - } else { - die "implement me"; - } -} - sub storage_info { my ($cfg, $content) = @_; my $ids = $cfg->{ids}; my $info = {}; - my $stypes = {}; my $slist = []; foreach my $storeid (keys %$ids) { @@ -2065,91 +630,28 @@ sub storage_info { avail => 0, used => 0, shared => $ids->{$storeid}->{shared} ? 1 : 0, - content => content_hash_to_string($ids->{$storeid}->{content}), + content => PVE::Storage::Plugin::content_hash_to_string($ids->{$storeid}->{content}), active => 0, }; - $stypes->{$type} = 1; - push @$slist, $storeid; } - my $session = {}; - my $mountdata = ''; - my $iscsi_sessions = {}; - my $vgs = {}; + my $cache = {}; - if ($stypes->{lvm}) { - $session->{vgs} = lvm_vgs(); - $vgs = $session->{vgs}; - } - if ($stypes->{nfs}) { - $mountdata = read_proc_mounts(); - $session->{mountdata} = $mountdata; - } - if ($stypes->{iscsi}) { - $iscsi_sessions = iscsi_session_list(); - $session->{iscsi_sessions} = $iscsi_sessions; - } - - eval { activate_storage_list ($cfg, $slist, $session); }; + eval { activate_storage_list($cfg, $slist, $cache); }; foreach my $storeid (keys %$ids) { my $scfg = $ids->{$storeid}; - next if !$info->{$storeid}; - my $type = $scfg->{type}; - - if ($type eq 'dir' || $type eq 'nfs') { - - my $path = $scfg->{path}; - - if ($type eq 'nfs') { - my $server = $scfg->{server}; - my $export = $scfg->{export}; - - next if !nfs_is_mounted ($server, $export, $path, $mountdata); - } - - my $timeout = 2; - my $res = PVE::Tools::df($path, $timeout); - - next if !$res || !$res->{total}; - - $info->{$storeid}->{total} = $res->{total}; - $info->{$storeid}->{avail} = $res->{avail}; - $info->{$storeid}->{used} = $res->{used}; - $info->{$storeid}->{active} = 1; - - } elsif ($type eq 'lvm') { - - my $vgname = $scfg->{vgname}; - - my $total = 0; - my $free = 0; - - if (defined ($vgs->{$vgname})) { - $total = $vgs->{$vgname}->{size}; - $free = $vgs->{$vgname}->{free}; - - $info->{$storeid}->{total} = $total; - $info->{$storeid}->{avail} = $free; - $info->{$storeid}->{used} = $total - $free; - $info->{$storeid}->{active} = 1; - } - - } elsif ($type eq 'iscsi') { - - $info->{$storeid}->{total} = 0; - $info->{$storeid}->{avail} = 0; - $info->{$storeid}->{used} = 0; - $info->{$storeid}->{active} = - defined ($iscsi_sessions->{$scfg->{target}}); - - } else { - die "implement me"; - } + my $plugin = PVE::Storage::Plugin->lookup($scfg->{type}); + my ($total, $avail, $used, $active) = $plugin->status($storeid, $scfg, $cache); + next if !$active; + $info->{$storeid}->{total} = $total; + $info->{$storeid}->{avail} = $avail; + $info->{$storeid}->{used} = $used; + $info->{$storeid}->{active} = $active; } return $info; @@ -2266,11 +768,11 @@ sub scan_iscsi { my ($portal_in) = @_; my $portal; - if (!($portal = resolv_portal ($portal_in))) { + if (!($portal = resolv_portal($portal_in))) { die "unable to parse/resolve portal address '${portal_in}'\n"; } - return iscsi_discovery($portal); + return PVE::Storage::ISCSIPlugin::iscsi_discovery($portal); } sub storage_default_format { @@ -2278,24 +780,14 @@ sub storage_default_format { my $scfg = storage_config ($cfg, $storeid); - my $def = $default_config->{$scfg->{type}}; - - my $def_format = 'raw'; - my $valid_formats = [ $def_format ]; - - if (defined ($def->{format})) { - $def_format = $scfg->{format} || $def->{format}->[1]; - $valid_formats = [ sort keys %{$def->{format}->[0]} ]; - } - - return wantarray ? ($def_format, $valid_formats) : $def_format; + return PVE::Storage::Plugin::default_format($scfg); } sub vgroup_is_used { my ($cfg, $vgname) = @_; foreach my $storeid (keys %{$cfg->{ids}}) { - my $scfg = storage_config ($cfg, $storeid); + my $scfg = storage_config($cfg, $storeid); if ($scfg->{type} eq 'lvm' && $scfg->{vgname} eq $vgname) { return 1; } @@ -2308,7 +800,7 @@ sub target_is_used { my ($cfg, $target) = @_; foreach my $storeid (keys %{$cfg->{ids}}) { - my $scfg = storage_config ($cfg, $storeid); + my $scfg = storage_config($cfg, $storeid); if ($scfg->{type} eq 'iscsi' && $scfg->{target} eq $target) { return 1; } @@ -2321,7 +813,7 @@ sub volume_is_used { my ($cfg, $volid) = @_; foreach my $storeid (keys %{$cfg->{ids}}) { - my $scfg = storage_config ($cfg, $storeid); + my $scfg = storage_config($cfg, $storeid); if ($scfg->{base} && $scfg->{base} eq $volid) { return 1; } @@ -2334,9 +826,9 @@ sub storage_is_used { my ($cfg, $storeid) = @_; foreach my $sid (keys %{$cfg->{ids}}) { - my $scfg = storage_config ($cfg, $sid); + my $scfg = storage_config($cfg, $sid); next if !$scfg->{base}; - my ($st) = parse_volume_id ($scfg->{base}); + my ($st) = parse_volume_id($scfg->{base}); return 1 if $st && $st eq $storeid; } @@ -2351,7 +843,7 @@ sub foreach_volid { foreach my $sid (keys %$list) { foreach my $info (@{$list->{$sid}}) { my $volid = $info->{volid}; - my ($sid1, $volname) = parse_volume_id ($volid, 1); + my ($sid1, $volname) = parse_volume_id($volid, 1); if ($sid1 && $sid1 eq $sid) { &$func ($volid, $sid, $info); } else { diff --git a/PVE/Storage/DirPlugin.pm b/PVE/Storage/DirPlugin.pm new file mode 100644 index 0000000..41842f2 --- /dev/null +++ b/PVE/Storage/DirPlugin.pm @@ -0,0 +1,58 @@ +package PVE::Storage::DirPlugin; + +use strict; +use warnings; +use File::Path; +use PVE::Storage::Plugin; +use PVE::JSONSchema qw(get_standard_option); + +use base qw(PVE::Storage::Plugin); + +# Configuration + +sub type { + return 'dir'; +} + +sub plugindata { + return { + content => [ { images => 1, rootdir => 1, vztmpl => 1, iso => 1, backup => 1, none => 1 }, + { images => 1, rootdir => 1 }], + format => [ { raw => 1, qcow2 => 1, vmdk => 1 } , 'raw' ], + }; +} + +sub properties { + return { + path => { + description => "File system path.", + type => 'string', format => 'pve-storage-path', + }, + }; +} + +sub options { + return { + path => { fixed => 1 }, + nodes => { optional => 1 }, + shared => { optional => 1 }, + disable => { optional => 1 }, + maxfiles => { optional => 1 }, + content => { optional => 1 }, + format => { optional => 1 }, + }; +} + +# Storage implementation + +sub activate_storage { + my ($class, $storeid, $scfg, $cache) = @_; + + my $path = $scfg->{path}; + mkpath $path; + + $class->SUPER::activate_storage($storeid, $scfg, $cache); +} + + +1; diff --git a/PVE/Storage/ISCSIPlugin.pm b/PVE/Storage/ISCSIPlugin.pm new file mode 100644 index 0000000..e76ccbf --- /dev/null +++ b/PVE/Storage/ISCSIPlugin.pm @@ -0,0 +1,374 @@ +package PVE::Storage::ISCSIPlugin; + +use strict; +use warnings; +use File::stat; +use IO::Dir; +use IO::File; +use PVE::Tools qw(run_command file_read_firstline trim dir_glob_regex dir_glob_foreach); +use PVE::Storage::Plugin; +use PVE::JSONSchema qw(get_standard_option); +use Net::Ping; + +use base qw(PVE::Storage::Plugin); + +# iscsi helper function + +my $ISCSIADM = '/usr/bin/iscsiadm'; +$ISCSIADM = undef if ! -X $ISCSIADM; + +sub check_iscsi_support { + my $noerr = shift; + + if (!$ISCSIADM) { + my $msg = "no iscsi support - please install open-iscsi"; + if ($noerr) { + warn "warning: $msg\n"; + return 0; + } + + die "error: $msg\n"; + } + + return 1; +} + +sub iscsi_session_list { + + check_iscsi_support (); + + my $cmd = [$ISCSIADM, '--mode', 'session']; + + my $res = {}; + + run_command($cmd, outfunc => sub { + my $line = shift; + + if ($line =~ m/^tcp:\s+\[(\S+)\]\s+\S+\s+(\S+)\s*$/) { + my ($session, $target) = ($1, $2); + # there can be several sessions per target (multipath) + push @{$res->{$target}}, $session; + + } + }); + + return $res; +} + +sub iscsi_test_portal { + my ($portal) = @_; + + my ($server, $port) = split(':', $portal); + my $p = Net::Ping->new("tcp", 2); + $p->port_number($port || 3260); + return $p->ping($server); +} + +sub iscsi_discovery { + my ($portal) = @_; + + check_iscsi_support (); + + my $cmd = [$ISCSIADM, '--mode', 'discovery', '--type', 'sendtargets', + '--portal', $portal]; + + my $res = {}; + + return $res if !iscsi_test_portal($portal); # fixme: raise exception here? + + run_command($cmd, outfunc => sub { + my $line = shift; + + if ($line =~ m/^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d+)\,\S+\s+(\S+)\s*$/) { + my $portal = $1; + my $target = $2; + # one target can have more than one portal (multipath). + push @{$res->{$target}}, $portal; + } + }); + + return $res; +} + +sub iscsi_login { + my ($target, $portal_in) = @_; + + check_iscsi_support (); + + eval { iscsi_discovery ($portal_in); }; + warn $@ if $@; + + my $cmd = [$ISCSIADM, '--mode', 'node', '--targetname', $target, '--login']; + run_command($cmd); +} + +sub iscsi_logout { + my ($target, $portal) = @_; + + check_iscsi_support (); + + my $cmd = [$ISCSIADM, '--mode', 'node', '--targetname', $target, '--logout']; + run_command($cmd); +} + +my $rescan_filename = "/var/run/pve-iscsi-rescan.lock"; + +sub iscsi_session_rescan { + my $session_list = shift; + + check_iscsi_support(); + + my $rstat = stat($rescan_filename); + + if (!$rstat) { + if (my $fh = IO::File->new($rescan_filename, "a")) { + utime undef, undef, $fh; + close($fh); + } + } else { + my $atime = $rstat->atime; + my $tdiff = time() - $atime; + # avoid frequent rescans + return if !($tdiff < 0 || $tdiff > 10); + utime undef, undef, $rescan_filename; + } + + foreach my $session (@$session_list) { + my $cmd = [$ISCSIADM, '--mode', 'session', '-r', $session, '-R']; + eval { run_command($cmd, outfunc => sub {}); }; + warn $@ if $@; + } +} + +sub load_stable_scsi_paths { + + my $stable_paths = {}; + + my $stabledir = "/dev/disk/by-id"; + + if (my $dh = IO::Dir->new($stabledir)) { + while (defined(my $tmp = $dh->read)) { + # exclude filenames with part in name (same disk but partitions) + # use only filenames with scsi(with multipath i have the same device + # with dm-uuid-mpath , dm-name and scsi in name) + if($tmp !~ m/-part\d+$/ && $tmp =~ m/^scsi-/) { + my $path = "$stabledir/$tmp"; + my $bdevdest = readlink($path); + if ($bdevdest && $bdevdest =~ m|^../../([^/]+)|) { + $stable_paths->{$1}=$tmp; + } + } + } + $dh->close; + } + return $stable_paths; +} + +sub iscsi_device_list { + + my $res = {}; + + my $dirname = '/sys/class/iscsi_session'; + + my $stable_paths = load_stable_scsi_paths(); + + dir_glob_foreach($dirname, 'session(\d+)', sub { + my ($ent, $session) = @_; + + my $target = file_read_firstline("$dirname/$ent/targetname"); + return if !$target; + + my (undef, $host) = dir_glob_regex("$dirname/$ent/device", 'target(\d+):.*'); + return if !defined($host); + + dir_glob_foreach("/sys/bus/scsi/devices", "$host:" . '(\d+):(\d+):(\d+)', sub { + my ($tmp, $channel, $id, $lun) = @_; + + my $type = file_read_firstline("/sys/bus/scsi/devices/$tmp/type"); + return if !defined($type) || $type ne '0'; # list disks only + + my $bdev; + if (-d "/sys/bus/scsi/devices/$tmp/block") { # newer kernels + (undef, $bdev) = dir_glob_regex("/sys/bus/scsi/devices/$tmp/block/", '([A-Za-z]\S*)'); + } else { + (undef, $bdev) = dir_glob_regex("/sys/bus/scsi/devices/$tmp", 'block:(\S+)'); + } + return if !$bdev; + + #check multipath + if (-d "/sys/block/$bdev/holders") { + my $multipathdev = dir_glob_regex("/sys/block/$bdev/holders", '[A-Za-z]\S*'); + $bdev = $multipathdev if $multipathdev; + } + + my $blockdev = $stable_paths->{$bdev}; + return if !$blockdev; + + my $size = file_read_firstline("/sys/block/$bdev/size"); + return if !$size; + + my $volid = "$channel.$id.$lun.$blockdev"; + + $res->{$target}->{$volid} = { + 'format' => 'raw', + 'size' => int($size * 512), + 'vmid' => 0, # not assigned to any vm + 'channel' => int($channel), + 'id' => int($id), + 'lun' => int($lun), + }; + + #print "TEST: $target $session $host,$bus,$tg,$lun $blockdev\n"; + }); + + }); + + return $res; +} + +# Configuration + +sub type { + return 'iscsi'; +} + +sub plugindata { + return { + content => [ {images => 1, none => 1}, { images => 1 }], + }; +} + +sub properties { + return { + target => { + description => "iSCSI target.", + type => 'string', + }, + portal => { + description => "iSCSI portal (IP or DNS name with optional port).", + type => 'string', format => 'pve-storage-portal-dns', + }, + }; +} + +sub options { + return { + portal => { fixed => 1 }, + target => { fixed => 1 }, + nodes => { optional => 1}, + disable => { optional => 1}, + content => { optional => 1}, + }; +} + +# Storage implementation + +sub parse_volname { + my ($class, $volname) = @_; + + if ($volname =~ m!^\d+\.\d+\.\d+\.(\S+)$!) { + return ('images', $1, undef); + } + + die "unable to parse iscsi volume name '$volname'\n"; +} + +sub path { + my ($class, $scfg, $volname) = @_; + + my ($vtype, $name, $vmid) = $class->parse_volname($volname); + + my $path = "/dev/disk/by-id/$name"; + + return ($path, $vmid, $vtype); +} + +sub alloc_image { + my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_; + + die "can't allocate space in iscsi storage\n"; +} + +sub free_image { + my ($class, $storeid, $scfg, $volname) = @_; + + die "can't free space in iscsi storage\n"; +} + +sub list_images { + my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_; + + my $res = []; + + $cache->{iscsi_devices} = iscsi_device_list() if !$cache->{iscsi_devices}; + + # we have no owner for iscsi devices + + my $target = $scfg->{target}; + + if (my $dat = $cache->{iscsi_devices}->{$target}) { + + foreach my $volname (keys %$dat) { + + my $volid = "$storeid:$volname"; + + if ($vollist) { + my $found = grep { $_ eq $volid } @$vollist; + next if !$found; + } else { + # we have no owner for iscsi devices + next if defined($vmid); + } + + my $info = $dat->{$volname}; + $info->{volid} = $volid; + + push @$res, $info; + } + } + + return $res; +} + +sub status { + my ($class, $storeid, $scfg, $cache) = @_; + + $cache->{iscsi_sessions} = iscsi_session_list() if !$cache->{iscsi_sessions}; + + my $active = defined($cache->{iscsi_sessions}->{$scfg->{target}}); + + return (0, 0, 0, $active); +} + +sub activate_storage { + my ($class, $storeid, $scfg, $cache) = @_; + + return if !check_iscsi_support(1); + + $cache->{iscsi_sessions} = iscsi_session_list() if !$cache->{iscsi_sessions}; + + my $iscsi_sess = $cache->{iscsi_sessions}->{$scfg->{target}}; + if (!defined ($iscsi_sess)) { + eval { iscsi_login($scfg->{target}, $scfg->{portal}); }; + warn $@ if $@; + } else { + # make sure we get all devices + iscsi_session_rescan($iscsi_sess); + } +} + +sub deactivate_storage { + my ($class, $storeid, $scfg, $cache) = @_; + + return if !check_iscsi_support(1); + + $cache->{iscsi_sessions} = iscsi_session_list() if !$cache->{iscsi_sessions}; + + my $iscsi_sess = $cache->{iscsi_sessions}->{$scfg->{target}}; + + if (defined ($iscsi_sess)) { + iscsi_logout($scfg->{target}, $scfg->{portal}); + } +} + +1; diff --git a/PVE/Storage/LVMPlugin.pm b/PVE/Storage/LVMPlugin.pm new file mode 100644 index 0000000..7f4e74c --- /dev/null +++ b/PVE/Storage/LVMPlugin.pm @@ -0,0 +1,411 @@ +package PVE::Storage::LVMPlugin; + +use strict; +use warnings; +use IO::File; +use PVE::Tools qw(run_command trim); +use PVE::Storage::Plugin; +use PVE::JSONSchema qw(get_standard_option); + +use base qw(PVE::Storage::Plugin); + +# lvm helper functions + +sub lvm_pv_info { + my ($device) = @_; + + die "no device specified" if !$device; + + my $has_label = 0; + + my $cmd = ['/usr/bin/file', '-L', '-s', $device]; + run_command($cmd, outfunc => sub { + my $line = shift; + $has_label = 1 if $line =~ m/LVM2/; + }); + + return undef if !$has_label; + + $cmd = ['/sbin/pvs', '--separator', ':', '--noheadings', '--units', 'k', + '--unbuffered', '--nosuffix', '--options', + 'pv_name,pv_size,vg_name,pv_uuid', $device]; + + my $pvinfo; + run_command($cmd, outfunc => sub { + my $line = shift; + + $line = trim($line); + + my ($pvname, $size, $vgname, $uuid) = split(':', $line); + + die "found multiple pvs entries for device '$device'\n" + if $pvinfo; + + $pvinfo = { + pvname => $pvname, + size => $size, + vgname => $vgname, + uuid => $uuid, + }; + }); + + return $pvinfo; +} + +sub clear_first_sector { + my ($dev) = shift; + + if (my $fh = IO::File->new($dev, "w")) { + my $buf = 0 x 512; + syswrite $fh, $buf; + $fh->close(); + } +} + +sub lvm_create_volume_group { + my ($device, $vgname, $shared) = @_; + + my $res = lvm_pv_info($device); + + if ($res->{vgname}) { + return if $res->{vgname} eq $vgname; # already created + die "device '$device' is already used by volume group '$res->{vgname}'\n"; + } + + clear_first_sector($device); # else pvcreate fails + + # we use --metadatasize 250k, which reseults in "pe_start = 512" + # so pe_start is aligned on a 128k boundary (advantage for SSDs) + my $cmd = ['/sbin/pvcreate', '--metadatasize', '250k', $device]; + + run_command($cmd, errmsg => "pvcreate '$device' error"); + + $cmd = ['/sbin/vgcreate', $vgname, $device]; + # push @$cmd, '-c', 'y' if $shared; # we do not use this yet + + run_command($cmd, errmsg => "vgcreate $vgname $device error"); +} + +sub lvm_vgs { + + my $cmd = ['/sbin/vgs', '--separator', ':', '--noheadings', '--units', 'b', + '--unbuffered', '--nosuffix', '--options', + 'vg_name,vg_size,vg_free']; + + my $vgs = {}; + eval { + run_command($cmd, outfunc => sub { + my $line = shift; + + $line = trim($line); + + my ($name, $size, $free) = split (':', $line); + + $vgs->{$name} = { size => int ($size), free => int ($free) }; + }); + }; + my $err = $@; + + # just warn (vgs return error code 5 if clvmd does not run) + # but output is still OK (list without clustered VGs) + warn $err if $err; + + return $vgs; +} + +sub lvm_lvs { + my ($vgname) = @_; + + my $cmd = ['/sbin/lvs', '--separator', ':', '--noheadings', '--units', 'b', + '--unbuffered', '--nosuffix', '--options', + 'vg_name,lv_name,lv_size,uuid,tags']; + + push @$cmd, $vgname if $vgname; + + my $lvs = {}; + run_command($cmd, outfunc => sub { + my $line = shift; + + $line = trim($line); + + my ($vg, $name, $size, $uuid, $tags) = split(':', $line); + + return if $name !~ m/^vm-(\d+)-/; + my $nid = $1; + + my $owner; + foreach my $tag (split (/,/, $tags)) { + if ($tag =~ m/^pve-vm-(\d+)$/) { + $owner = $1; + last; + } + } + + if ($owner) { + if ($owner ne $nid) { + warn "owner mismatch name = $name, owner = $owner\n"; + } + + $lvs->{$vg}->{$name} = { format => 'raw', size => $size, + uuid => $uuid, tags => $tags, + vmid => $owner }; + } + }); + + return $lvs; +} + +# Configuration + +PVE::JSONSchema::register_format('pve-storage-vgname', \&parse_lvm_name); +sub parse_lvm_name { + my ($name, $noerr) = @_; + + if ($name !~ m/^[a-z][a-z0-9\-\_\.]*[a-z0-9]$/i) { + return undef if $noerr; + die "lvm name '$name' contains illegal characters\n"; + } + + return $name; +} + +sub type { + return 'lvm'; +} + +sub plugindata { + return { + content => [ {images => 1}, { images => 1 }], + }; +} + +sub properties { + return { + vgname => { + description => "Volume group name.", + type => 'string', format => 'pve-storage-vgname', + }, + base => { + description => "Base volume. This volume is automatically activated.", + type => 'string', format => 'pve-volume-id', + }, + saferemove => { + description => "Zero-out data when removing LVs.", + type => 'boolean', + }, + }; +} + +sub options { + return { + vgname => { fixed => 1 }, + nodes => { optional => 1 }, + shared => { optional => 1 }, + disable => { optional => 1 }, + saferemove => { optional => 1 }, + content => { optional => 1 }, + base => { fixed => 1, optional => 1 }, + }; +} + +# Storage implementation + +sub parse_volname { + my ($class, $volname) = @_; + + parse_lvm_name($volname); + + if ($volname =~ m/^(vm-(\d+)-\S+)$/) { + return ('images', $1, $2); + } + + die "unable to parse lvm volume name '$volname'\n"; +} + +sub path { + my ($class, $scfg, $volname) = @_; + + my ($vtype, $name, $vmid) = $class->parse_volname($volname); + + my $vg = $scfg->{vgname}; + + my $path = "/dev/$vg/$name"; + + return ($path, $vmid, $vtype); +} + +sub alloc_image { + my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_; + + die "unsupported format '$fmt'" if $fmt ne 'raw'; + + die "illegal name '$name' - sould be 'vm-$vmid-*'\n" + if $name && $name !~ m/^vm-$vmid-/; + + my $vgs = lvm_vgs(); + + my $vg = $scfg->{vgname}; + + die "no such volume gruoup '$vg'\n" if !defined ($vgs->{$vg}); + + my $free = int($vgs->{$vg}->{free}); + + die "not enough free space ($free < $size)\n" if $free < $size; + + if (!$name) { + my $lvs = lvm_lvs($vg); + + for (my $i = 1; $i < 100; $i++) { + my $tn = "vm-$vmid-disk-$i"; + if (!defined ($lvs->{$vg}->{$tn})) { + $name = $tn; + last; + } + } + } + + die "unable to allocate an image name for VM $vmid in storage '$storeid'\n" + if !$name; + + my $cmd = ['/sbin/lvcreate', '-aly', '--addtag', "pve-vm-$vmid", '--size', "${size}k", '--name', $name, $vg]; + + run_command($cmd, errmsg => "lvcreate '$vg/pve-vm-$vmid' error"); + + return $name; +} + +sub free_image { + my ($class, $storeid, $scfg, $volname) = @_; + + my $vg = $scfg->{vgname}; + + # we need to zero out LVM data for security reasons + # and to allow thin provisioning + + my $zero_out_worker = sub { + print "zero-out data on image $volname\n"; + my $cmd = ['dd', "if=/dev/zero", "of=/dev/$vg/del-$volname", "bs=1M"]; + eval { run_command($cmd, errmsg => "zero out failed"); }; + warn $@ if $@; + + $class->cluster_lock_storage($storeid, $scfg->{shared}, undef, sub { + my $cmd = ['/sbin/lvremove', '-f', "$vg/del-$volname"]; + run_command($cmd, errmsg => "lvremove '$vg/del-$volname' error"); + }); + print "successfully removed volume $volname\n"; + }; + + if ($scfg->{saferemove}) { + # avoid long running task, so we only rename here + my $cmd = ['/sbin/lvrename', $vg, $volname, "del-$volname"]; + run_command($cmd, errmsg => "lvrename '$vg/$volname' error"); + return $zero_out_worker; + } else { + my $tmpvg = $scfg->{vgname}; + my $cmd = ['/sbin/lvremove', '-f', "$tmpvg/$volname"]; + run_command($cmd, errmsg => "lvremove '$tmpvg/$volname' error"); + } + + return undef; +} + +sub list_images { + my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_; + + my $vgname = $scfg->{vgname}; + + $cache->{lvs} = lvm_lvs() if !$cache->{lvs}; + + my $res = []; + + if (my $dat = $cache->{lvs}->{$vgname}) { + + foreach my $volname (keys %$dat) { + + my $owner = $dat->{$volname}->{vmid}; + + my $volid = "$storeid:$volname"; + + if ($vollist) { + my $found = grep { $_ eq $volid } @$vollist; + next if !$found; + } else { + next if defined ($vmid) && ($owner ne $vmid); + } + + my $info = $dat->{$volname}; + $info->{volid} = $volid; + + push @$res, $info; + } + } + + return $res; +} + +sub status { + my ($class, $storeid, $scfg, $cache) = @_; + + $cache->{vgs} = lvm_vgs() if !$cache->{vgs}; + + my $vgname = $scfg->{vgname}; + + my $total = 0; + my $free = 0; + my $used = 0; + + if (my $info = $cache->{vgs}->{$vgname}) { + return ($info->{size}, $info->{free}, $total - $free, 1); + } + + return undef; +} + +sub activate_storage { + my ($class, $storeid, $scfg, $cache) = @_; + + $cache->{vgs} = lvm_vgs() if !$cache->{vgs}; + + # In LVM2, vgscans take place automatically; + # this is just to be sure + if ($cache->{vgs} && !$cache->{vgscaned} && + !$cache->{vgs}->{$scfg->{vgname}}) { + $cache->{vgscaned} = 1; + my $cmd = ['/sbin/vgscan', '--ignorelockingfailure', '--mknodes']; + eval { run_command($cmd, outfunc => sub {}); }; + warn $@ if $@; + } + + # we do not acticate any volumes here ('vgchange -aly') + # instead, volumes are activate individually later +} + +sub deactivate_storage { + my ($class, $storeid, $scfg, $cache) = @_; + + my $cmd = ['/sbin/vgchange', '-aln', $scfg->{vgname}]; + run_command($cmd, errmsg => "can't deactivate VG '$scfg->{vgname}'"); +} + +sub activate_volume { + my ($class, $storeid, $scfg, $volname, $exclusive, $cache) = @_; + + my $path = $class->path($scfg, $volname); + + my $lvm_activate_mode = $exclusive ? 'ey' : 'ly'; + + my $cmd = ['/sbin/lvchange', "-a$lvm_activate_mode", $path]; + run_command($cmd, errmsg => "can't activate LV '$path'"); +} + +sub deactivate_volume { + my ($class, $storeid, $scfg, $volname, $cache) = @_; + + my $path = $class->path($scfg, $volname); + return if ! -b $path; + + my $cmd = ['/sbin/lvchange', '-aln', $path]; + run_command($cmd, errmsg => "can't deactivate LV '$path'"); +} + +1; diff --git a/PVE/Storage/Makefile b/PVE/Storage/Makefile new file mode 100644 index 0000000..1d2322f --- /dev/null +++ b/PVE/Storage/Makefile @@ -0,0 +1,5 @@ +SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm ISCSIPlugin.pm + +.PHONY: install +install: + for i in ${SOURCES}; do install -D -m 0644 $$i ${DESTDIR}${PERLDIR}/PVE/Storage/$$i; done diff --git a/PVE/Storage/NFSPlugin.pm b/PVE/Storage/NFSPlugin.pm new file mode 100644 index 0000000..88aa4e4 --- /dev/null +++ b/PVE/Storage/NFSPlugin.pm @@ -0,0 +1,163 @@ +package PVE::Storage::NFSPlugin; + +use strict; +use warnings; +use IO::File; +use PVE::Storage::Plugin; +use PVE::JSONSchema qw(get_standard_option); + +use base qw(PVE::Storage::Plugin); + +# NFS helper functions + +sub read_proc_mounts { + + local $/; # enable slurp mode + + my $data = ""; + if (my $fd = IO::File->new("/proc/mounts", "r")) { + $data = <$fd>; + close ($fd); + } + + return $data; +} + +sub nfs_is_mounted { + my ($server, $export, $mountpoint, $mountdata) = @_; + + my $source = "$server:$export"; + + $mountdata = read_proc_mounts() if !$mountdata; + + if ($mountdata =~ m|^$source/?\s$mountpoint\snfs|m) { + return $mountpoint; + } + + return undef; +} + +sub nfs_mount { + my ($server, $export, $mountpoint, $options) = @_; + + my $source = "$server:$export"; + + my $cmd = ['/bin/mount', '-t', 'nfs', $source, $mountpoint]; + if ($options) { + push @$cmd, '-o', $options; + } + + run_command($cmd, errmsg => "mount error"); +} + +# Configuration + +sub type { + return 'nfs'; +} + +sub plugindata { + return { + content => [ { images => 1, rootdir => 1, vztmpl => 1, iso => 1, backup => 1}, + { images => 1 }], + format => [ { raw => 1, qcow2 => 1, vmdk => 1 } , 'raw' ], + }; +} + +sub properties { + return { + export => { + description => "NFS export path.", + type => 'string', format => 'pve-storage-path', + }, + server => { + description => "Server IP or DNS name.", + type => 'string', format => 'pve-storage-server', + }, + options => { + description => "NFS mount options (see 'man nfs')", + type => 'string', format => 'pve-storage-options', + }, + }; +} + +sub options { + return { + path => { fixed => 1 }, + server => { fixed => 1 }, + export => { fixed => 1 }, + nodes => { optional => 1 }, + disable => { optional => 1 }, + maxfiles => { optional => 1 }, + options => { optional => 1 }, + content => { optional => 1 }, + format => { optional => 1 }, + }; +} + + +sub check_config { + my ($class, $sectionId, $config, $create, $skipSchemaCheck) = @_; + + $config->{path} = "/mnt/pve/$sectionId" if $create && !$config->{path}; + + return $class->SUPER::check_config($sectionId, $config, $create, $skipSchemaCheck); +} + +# Storage implementation + +sub status { + my ($class, $storeid, $scfg, $cache) = @_; + + $cache->{mountdata} = read_proc_mounts() if !$cache->{mountdata}; + + my $path = $scfg->{path}; + my $server = $scfg->{server}; + my $export = $scfg->{export}; + + return undef if !nfs_is_mounted($server, $export, $path, $cache->{mountdata}); + + return $class->SUPER::status($storeid, $scfg, $cache); +} + +sub activate_storage { + my ($class, $storeid, $scfg, $cache) = @_; + + $cache->{mountdata} = read_proc_mounts() if !$cache->{mountdata}; + + my $path = $scfg->{path}; + my $server = $scfg->{server}; + my $export = $scfg->{export}; + + if (!nfs_is_mounted($server, $export, $path, $cache->{mountdata})) { + + # NOTE: only call mkpath when not mounted (avoid hang + # when NFS server is offline + + mkpath $path; + + die "unable to activate storage '$storeid' - " . + "directory '$path' does not exist\n" if ! -d $path; + + nfs_mount($server, $export, $path, $scfg->{options}); + } + + $class->SUPER::activate_storage($storeid, $scfg, $cache); +} + +sub deactivate_storage { + my ($class, $storeid, $scfg, $cache) = @_; + + $cache->{mountdata} = read_proc_mounts() if !$cache->{mountdata}; + + my $path = $scfg->{path}; + my $server = $scfg->{server}; + my $export = $scfg->{export}; + + if (nfs_is_mounted($server, $export, $path, $cache->{mountdata})) { + my $cmd = ['/bin/umount', $path]; + run_command($cmd, errmsg => 'umount error'); + } +} + +1; diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm new file mode 100644 index 0000000..71ed6a1 --- /dev/null +++ b/PVE/Storage/Plugin.pm @@ -0,0 +1,582 @@ +package PVE::Storage::Plugin; + +use strict; +use warnings; +use File::Path; +use PVE::Tools qw(run_command); +use PVE::JSONSchema qw(get_standard_option); +use PVE::Cluster qw(cfs_register_file); + +use Data::Dumper; + +use base qw(PVE::SectionConfig); + +cfs_register_file ('storage.cfg', + sub { __PACKAGE__->parse_config(@_); }, + sub { __PACKAGE__->write_config(@_); }); + +my $defaultData = { + propertyList => { + type => { description => "Storage type." }, + storage => get_standard_option('pve-storage-id'), + nodes => get_standard_option('pve-node-list', { optional => 1 }), + content => { + description => "Allowed content types.", + type => 'string', format => 'pve-storage-content-list', + optional => 1, + }, + disable => { + description => "Flag to disable the storage.", + type => 'boolean', + optional => 1, + }, + maxfiles => { + description => "Maximal number of backup files per VM. Use '0' for unlimted.", + type => 'integer', + minimum => 0, + optional => 1, + }, + shared => { + description => "Mark storage as shared.", + type => 'boolean', + optional => 1, + }, + 'format' => { + description => "Default Image format.", + type => 'string', format => 'pve-storage-format', + optional => 1, + }, + }, +}; + +sub content_hash_to_string { + my $hash = shift; + + my @cta; + foreach my $ct (keys %$hash) { + push @cta, $ct if $hash->{$ct}; + } + + return join(',', @cta); +} + +sub valid_content_types { + my ($type) = @_; + + my $def = $defaultData->{plugindata}->{$type}; + + return {} if !$def; + + return $def->{content}->[0]; +} + +sub default_format { + my ($scfg) = @_; + + my $type = $scfg->{type}; + my $def = $defaultData->{plugindata}->{$type}; + + my $def_format = 'raw'; + my $valid_formats = [ $def_format ]; + + if (defined($def->{format})) { + $def_format = $scfg->{format} || $def->{format}->[1]; + $valid_formats = [ sort keys %{$def->{format}->[0]} ]; + } + + return wantarray ? ($def_format, $valid_formats) : $def_format; +} + +PVE::JSONSchema::register_format('pve-storage-path', \&verify_path); +sub verify_path { + my ($path, $noerr) = @_; + + # fixme: exclude more shell meta characters? + # we need absolute paths + if ($path !~ m|^/[^;\(\)]+|) { + return undef if $noerr; + die "value does not look like a valid absolute path\n"; + } + return $path; +} + +PVE::JSONSchema::register_format('pve-storage-server', \&verify_server); +sub verify_server { + my ($server, $noerr) = @_; + + # fixme: use better regex ? + # IP or DNS name + if ($server !~ m/^[[:alnum:]\-\.]+$/) { + return undef if $noerr; + die "value does not look like a valid server name or IP address\n"; + } + return $server; +} + +# fixme: do we need this +#PVE::JSONSchema::register_format('pve-storage-portal', \&verify_portal); +#sub verify_portal { +# my ($portal, $noerr) = @_; +# +# # IP with optional port +# if ($portal !~ m/^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(:\d+)?$/) { +# return undef if $noerr; +# die "value does not look like a valid portal address\n"; +# } +# return $portal; +#} + +PVE::JSONSchema::register_format('pve-storage-portal-dns', \&verify_portal_dns); +sub verify_portal_dns { + my ($portal, $noerr) = @_; + + # IP or DNS name with optional port + if ($portal !~ m/^(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}|[[:alnum:]\-\.]+)(:\d+)?$/) { + return undef if $noerr; + die "value does not look like a valid portal address\n"; + } + return $portal; +} + +PVE::JSONSchema::register_format('pve-storage-content', \&verify_content); +sub verify_content { + my ($ct, $noerr) = @_; + + my $valid_content = valid_content_types('dir'); # dir includes all types + + if (!$valid_content->{$ct}) { + return undef if $noerr; + die "invalid content type '$ct'\n"; + } + + return $ct; +} + +PVE::JSONSchema::register_format('pve-storage-format', \&verify_format); +sub verify_format { + my ($fmt, $noerr) = @_; + + if ($fmt !~ m/(raw|qcow2|vmdk)/) { + return undef if $noerr; + die "invalid format '$fmt'\n"; + } + + return $fmt; +} + +PVE::JSONSchema::register_format('pve-storage-options', \&verify_options); +sub verify_options { + my ($value, $noerr) = @_; + + # mount options (see man fstab) + if ($value !~ m/^\S+$/) { + return undef if $noerr; + die "invalid options '$value'\n"; + } + + return $value; +} + + +sub private { + return $defaultData; +} + +sub parse_section_header { + my ($class, $line) = @_; + + if ($line =~ m/^(\S+):\s*(\S+)\s*$/) { + my ($type, $storeid) = (lc($1), $2); + my $errmsg = undef; # set if you want to skip whole section + eval { PVE::JSONSchema::parse_storage_id($storeid); }; + $errmsg = $@ if $@; + my $config = {}; # to return additional attributes + return ($type, $storeid, $errmsg, $config); + } + return undef; +} + +sub decode_value { + my ($class, $type, $key, $value) = @_; + + my $def = $defaultData->{plugindata}->{$type}; + + if ($key eq 'content') { + my $valid_content = $def->{content}->[0]; + + my $res = {}; + + foreach my $c (PVE::Tools::split_list($value)) { + if (!$valid_content->{$c}) { + die "storage does not support content type '$c'\n"; + } + $res->{$c} = 1; + } + + if ($res->{none} && scalar (keys %$res) > 1) { + die "unable to combine 'none' with other content types\n"; + } + + return $res; + } elsif ($key eq 'format') { + my $valid_formats = $def->{format}->[0]; + + if (!$valid_formats->{$value}) { + die "storage does not support format '$value'\n"; + } + + return $value; + } elsif ($key eq 'nodes') { + my $res = {}; + + foreach my $node (PVE::Tools::split_list($value)) { + if (PVE::JSONSchema::pve_verify_node_name($node)) { + $res->{$node} = 1; + } + } + + # fixme: + # no node restrictions for local storage + #if ($storeid && $storeid eq 'local' && scalar(keys(%$res))) { + # die "storage '$storeid' does not allow node restrictions\n"; + #} + + return $res; + } + + return $value; +} + +sub encode_value { + my ($class, $type, $key, $value) = @_; + + if ($key eq 'nodes') { + return join(',', keys(%$value)); + } elsif ($key eq 'content') { + my $res = content_hash_to_string($value) || 'none'; + return $res; + } + + return $value; +} + +sub parse_config { + my ($class, $filename, $raw) = @_; + + my $cfg = $class->SUPER::parse_config($filename, $raw); + my $ids = $cfg->{ids}; + + # make sure we have a reasonable 'local:' storage + # openvz expects things to be there + if (!$ids->{local} || $ids->{local}->{type} ne 'dir' || + ($ids->{local}->{path} && $ids->{local}->{path} ne '/var/lib/vz')) { + $ids->{local} = { + type => 'dir', + priority => 0, # force first entry + path => '/var/lib/vz', + maxfiles => 0, + content => { images => 1, rootdir => 1, vztmpl => 1, iso => 1}, + }; + } + + # we always need this for OpenVZ + $ids->{local}->{content}->{rootdir} = 1; + $ids->{local}->{content}->{vztmpl} = 1; + delete ($ids->{local}->{disable}); + + # make sure we have a path + $ids->{local}->{path} = '/var/lib/vz' if !$ids->{local}->{path}; + + # remove node restrictions for local storage + delete($ids->{local}->{nodes}); + + foreach my $storeid (keys %$ids) { + my $d = $ids->{$storeid}; + my $type = $d->{type}; + + my $def = $defaultData->{plugindata}->{$type}; + + if ($def->{content}) { + $d->{content} = $def->{content}->[1] if !$d->{content}; + } + + if ($type eq 'iscsi' || $type eq 'nfs') { + $d->{shared} = 1; + } + } + + return $cfg; +} + +# Storage implementation + +sub cluster_lock_storage { + my ($class, $storeid, $shared, $timeout, $func, @param) = @_; + + my $res; + if (!$shared) { + my $lockid = "pve-storage-$storeid"; + my $lockdir = "/var/lock/pve-manager"; + mkdir $lockdir; + $res = PVE::Tools::lock_file("$lockdir/$lockid", $timeout, $func, @param); + die $@ if $@; + } else { + $res = PVE::Cluster::cfs_lock_storage($storeid, $timeout, $func, @param); + die $@ if $@; + } + return $res; +} + +sub parse_name_dir { + my $name = shift; + + if ($name =~ m!^([^/\s]+\.(raw|qcow2|vmdk))$!) { + return ($1, $2); + } + + die "unable to parse volume filename '$name'\n"; +} + +sub parse_volname { + my ($class, $volname) = @_; + + if ($volname =~ m!^(\d+)/(\S+)$!) { + my ($vmid, $name) = ($1, $2); + parse_name_dir($name); + return ('images', $name, $vmid); + } elsif ($volname =~ m!^iso/([^/]+\.[Ii][Ss][Oo])$!) { + return ('iso', $1); + } elsif ($volname =~ m!^vztmpl/([^/]+\.tar\.gz)$!) { + return ('vztmpl', $1); + } elsif ($volname =~ m!^rootdir/(\d+)$!) { + return ('rootdir', $1, $1); + } elsif ($volname =~ m!^backup/([^/]+(\.(tar|tar\.gz|tar\.lzo|tgz)))$!) { + my $fn = $1; + if ($fn =~ m/^vzdump-(openvz|qemu)-(\d+)-.+/) { + return ('backup', $fn, $2); + } + return ('backup', $fn); + } + + die "unable to parse directory volume name '$volname'\n"; +} + +my $vtype_subdirs = { + images => 'images', + rootdir => 'private', + iso => 'template/iso', + vztmpl => 'template/cache', + backup => 'dump', +}; + +sub get_subdir { + my ($class, $scfg, $vtype) = @_; + + my $path = $scfg->{path}; + + die "storage definintion has no path\n" if !$path; + + my $subdir = $vtype_subdirs->{$vtype}; + + die "unknown vtype '$vtype'\n" if !defined($subdir); + + return "$path/$subdir"; +} + +sub path { + my ($class, $scfg, $volname) = @_; + + my ($vtype, $name, $vmid) = $class->parse_volname($volname); + + my $dir = $class->get_subdir($scfg, $vtype); + + $dir .= "/$vmid" if $vtype eq 'images'; + + my $path = "$dir/$name"; + + return wantarray ? ($path, $vmid, $vtype) : $path; +} + +sub alloc_image { + my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_; + + my $imagedir = $class->get_subdir($scfg, 'images'); + $imagedir .= "/$vmid"; + + mkpath $imagedir; + + if (!$name) { + for (my $i = 1; $i < 100; $i++) { + my @gr = <$imagedir/vm-$vmid-disk-$i.*>; + if (!scalar(@gr)) { + $name = "vm-$vmid-disk-$i.$fmt"; + last; + } + } + } + + die "unable to allocate an image name for VM $vmid in storage '$storeid'\n" + if !$name; + + my (undef, $tmpfmt) = parse_name_dir($name); + + die "illegal name '$name' - wrong extension for format ('$tmpfmt != '$fmt')\n" + if $tmpfmt ne $fmt; + + my $path = "$imagedir/$name"; + + die "disk image '$path' already exists\n" if -e $path; + + run_command("/usr/bin/qemu-img create -f $fmt '$path' ${size}K", + errmsg => "unable to create image"); + + return "$vmid/$name"; +} + +sub free_image { + my ($class, $storeid, $scfg, $volname) = @_; + + my $path = $class->path($scfg, $volname); + + if (! -f $path) { + warn "disk image '$path' does not exists\n"; + } else { + unlink $path; + } + + return undef; +} + +sub file_size_info { + my ($filename, $timeout) = @_; + + my $cmd = ['/usr/bin/qemu-img', 'info', $filename]; + + my $format; + my $size = 0; + my $used = 0; + + eval { + run_command($cmd, timeout => $timeout, outfunc => sub { + my $line = shift; + + if ($line =~ m/^file format:\s+(\S+)\s*$/) { + $format = $1; + } elsif ($line =~ m/^virtual size:\s\S+\s+\((\d+)\s+bytes\)$/) { + $size = int($1); + } elsif ($line =~ m/^disk size:\s+(\d+(.\d+)?)([KMGT])\s*$/) { + $used = $1; + my $u = $3; + + $used *= 1024 if $u eq 'K'; + $used *= (1024*1024) if $u eq 'M'; + $used *= (1024*1024*1024) if $u eq 'G'; + $used *= (1024*1024*1024*1024) if $u eq 'T'; + + $used = int($used); + } + }); + }; + + return wantarray ? ($size, $format, $used) : $size; +} + +sub list_images { + my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_; + + my $imagedir = $class->get_subdir($scfg, 'images'); + + my ($defFmt, $vaidFmts) = default_format($scfg); + my $fmts = join ('|', @$vaidFmts); + + my $res = []; + + foreach my $fn (<$imagedir/[0-9][0-9]*/*>) { + + next if $fn !~ m!^(/.+/(\d+)/([^/]+\.($fmts)))$!; + $fn = $1; # untaint + + my $owner = $2; + my $name = $3; + my $volid = "$storeid:$owner/$name"; + + if ($vollist) { + my $found = grep { $_ eq $volid } @$vollist; + next if !$found; + } else { + next if defined($vmid) && ($owner ne $vmid); + } + + my ($size, $format, $used) = file_size_info($fn); + + if ($format && $size) { + push @$res, { + volid => $volid, format => $format, + size => $size, vmid => $owner, used => $used }; + } + + } + + return $res; +} + +sub status { + my ($class, $storeid, $scfg, $cache) = @_; + + my $path = $scfg->{path}; + + die "storage definintion has no path\n" if !$path; + + my $timeout = 2; + my $res = PVE::Tools::df($path, $timeout); + + return undef if !$res || !$res->{total}; + + return ($res->{total}, $res->{avail}, $res->{used}, 1); +} + +sub activate_storage { + my ($class, $storeid, $scfg, $cache) = @_; + + my $path = $scfg->{path}; + + die "storage definintion has no path\n" if !$path; + + die "unable to activate storage '$storeid' - " . + "directory '$path' does not exist\n" if ! -d $path; + + if (defined($scfg->{content})) { + foreach my $vtype (keys %$vtype_subdirs) { + next if !defined($scfg->{content}->{$vtype}); + my $subdir = $class->get_subdir($scfg, $vtype); + mkpath $subdir if $subdir ne $path; + } + } +} + +sub deactivate_storage { + my ($class, $storeid, $scfg, $cache) = @_; + + # do nothing by default +} + +sub activate_volume { + my ($class, $storeid, $scfg, $volname, $exclusive, $cache) = @_; + + my $path = $class->path($scfg, $volname); + + # check is volume exists + if ($scfg->{path}) { + die "volume '$storeid:$volname' does not exist\n" if ! -e $path; + } else { + die "volume '$storeid:$volname' does not exist\n" if ! -b $path; + } +} + +sub deactivate_volume { + my ($class, $storeid, $scfg, $volname, $cache) = @_; + + # do nothing by default +} + +1;