From e34ce1444359ee06f50dd6907c0937d10748ce05 Mon Sep 17 00:00:00 2001 From: Alwin Antreich Date: Wed, 4 Jul 2018 12:43:31 +0200 Subject: [PATCH] Cephfs storage plugin - ability to mount through kernel and fuse client - allow mount options - get MONs from ceph config if not in storage.cfg - allow the use of ceph config with fuse client - Delete secret on cephfs storage creation Signed-off-by: Alwin Antreich --- PVE/Storage.pm | 2 + PVE/Storage/CephFSPlugin.pm | 194 ++++++++++++++++++++++++++++++++++++ PVE/Storage/CephTools.pm | 102 +++++++++++++++++++ PVE/Storage/Makefile | 2 +- PVE/Storage/Plugin.pm | 1 + debian/control | 1 + 6 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 PVE/Storage/CephFSPlugin.pm diff --git a/PVE/Storage.pm b/PVE/Storage.pm index d733380..f9732fe 100755 --- a/PVE/Storage.pm +++ b/PVE/Storage.pm @@ -28,6 +28,7 @@ use PVE::Storage::NFSPlugin; use PVE::Storage::CIFSPlugin; use PVE::Storage::ISCSIPlugin; use PVE::Storage::RBDPlugin; +use PVE::Storage::CephFSPlugin; use PVE::Storage::SheepdogPlugin; use PVE::Storage::ISCSIDirectPlugin; use PVE::Storage::GlusterfsPlugin; @@ -46,6 +47,7 @@ PVE::Storage::NFSPlugin->register(); PVE::Storage::CIFSPlugin->register(); PVE::Storage::ISCSIPlugin->register(); PVE::Storage::RBDPlugin->register(); +PVE::Storage::CephFSPlugin->register(); PVE::Storage::SheepdogPlugin->register(); PVE::Storage::ISCSIDirectPlugin->register(); PVE::Storage::GlusterfsPlugin->register(); diff --git a/PVE/Storage/CephFSPlugin.pm b/PVE/Storage/CephFSPlugin.pm new file mode 100644 index 0000000..8829e67 --- /dev/null +++ b/PVE/Storage/CephFSPlugin.pm @@ -0,0 +1,194 @@ +package PVE::Storage::CephFSPlugin; + +use strict; +use warnings; +use IO::File; +use Net::IP; +use File::Path; +use PVE::Tools qw(run_command); +use PVE::ProcFSTools; +use PVE::Storage::Plugin; +use PVE::JSONSchema qw(get_standard_option); +use PVE::Storage::CephTools; + +use base qw(PVE::Storage::Plugin); + +sub cephfs_is_mounted { + my ($scfg, $storeid, $mountdata) = @_; + + my $cmd_option = PVE::Storage::CephTools::ceph_connect_option($scfg, $storeid); + my $configfile = $cmd_option->{ceph_conf}; + my $server = $cmd_option->{mon_host} // PVE::Storage::CephTools::get_monaddr_list($configfile); + + my $subdir = $scfg->{subdir} // '/'; + my $mountpoint = $scfg->{path}; + my $source = "$server:$subdir"; + + $mountdata = PVE::ProcFSTools::parse_proc_mounts() if !$mountdata; + return $mountpoint if grep { + $_->[2] =~ m#^ceph|fuse\.ceph-fuse# && + $_->[0] =~ m#^\Q$source\E|ceph-fuse$# && + $_->[1] eq $mountpoint + } @$mountdata; + + warn "A filesystem is already mounted on $mountpoint\n" + if grep { $_->[1] eq $mountpoint } @$mountdata; + + return undef; +} + +sub cephfs_mount { + my ($scfg, $storeid) = @_; + + my $cmd; + my $mountpoint = $scfg->{path}; + my $subdir = $scfg->{subdir} // '/'; + + my $cmd_option = PVE::Storage::CephTools::ceph_connect_option($scfg, $storeid); + my $configfile = $cmd_option->{ceph_conf}; + my $secretfile = $cmd_option->{keyring}; + my $server = $cmd_option->{mon_host} // PVE::Storage::CephTools::get_monaddr_list($configfile); + + # fuse -> client-enforced quotas (kernel doesn't), updates w/ ceph-fuse pkg + # kernel -> better performance, less frequent updates + if ($scfg->{fuse}) { + # FIXME: ceph-fuse client complains about missing ceph.conf or keyring if + # not provided on its default locations but still connects. Fix upstream?? + $cmd = ['/usr/bin/ceph-fuse', '-n', "client.$cmd_option->{userid}", '-m', $server]; + push @$cmd, '--keyfile', $secretfile if defined($secretfile); + push @$cmd, '-r', $subdir if !($subdir =~ m|^/$|); + push @$cmd, $mountpoint; + push @$cmd, '--conf', $configfile if defined($configfile); + }else { + my $source = "$server:$subdir"; + $cmd = ['/bin/mount', '-t', 'ceph', $source, $mountpoint, '-o', "name=$cmd_option->{userid}"]; + push @$cmd, '-o', "secretfile=$secretfile" if defined($secretfile); + } + + if ($scfg->{options}) { + push @$cmd, '-o', $scfg->{options}; + } + + run_command($cmd, errmsg => "mount error"); +} + +# Configuration + +sub type { + return 'cephfs'; +} + +sub plugindata { + return { + content => [ { vztmpl => 1, iso => 1, backup => 1}, + { backup => 1 }], + }; +} + +sub properties { + return { + fuse => { + description => "Mount CephFS through FUSE.", + type => 'boolean', + }, + subdir => { + description => "Subdir to mount.", + type => 'string', format => 'pve-storage-path', + }, + }; +} + +sub options { + return { + path => { fixed => 1 }, + monhost => { optional => 1}, + nodes => { optional => 1 }, + subdir => { optional => 1 }, + disable => { optional => 1 }, + options => { optional => 1 }, + username => { optional => 1 }, + content => { optional => 1 }, + format => { optional => 1 }, + mkdir => { optional => 1 }, + fuse => { optional => 1 }, + bwlimit => { optional => 1 }, + }; +} + +sub check_config { + my ($class, $sectionId, $config, $create, $skipSchemaCheck) = @_; + + $config->{path} = "/mnt/pve/$sectionId" if $create && !$config->{path}; + + return $class->SUPER::check_config($sectionId, $config, $create, $skipSchemaCheck); +} + +# Storage implementation + +sub on_add_hook { + my ($class, $storeid, $scfg, %param) = @_; + + return if defined($scfg->{monhost}); # nothing to do if not pve managed ceph + + PVE::Storage::CephTools::ceph_create_keyfile($scfg->{type}, $storeid); +} + +sub on_delete_hook { + my ($class, $storeid, $scfg) = @_; + + return if defined($scfg->{monhost}); # nothing to do if not pve managed ceph + + PVE::Storage::CephTools::ceph_remove_keyfile($scfg->{type}, $storeid); + +} + +sub status { + my ($class, $storeid, $scfg, $cache) = @_; + + $cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts() + if !$cache->{mountdata}; + + return undef if !cephfs_is_mounted($scfg, $storeid, $cache->{mountdata}); + + return $class->SUPER::status($storeid, $scfg, $cache); +} + +sub activate_storage { + my ($class, $storeid, $scfg, $cache) = @_; + + $cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts() + if !$cache->{mountdata}; + + my $path = $scfg->{path}; + + if (!cephfs_is_mounted($scfg, $storeid, $cache->{mountdata})) { + + # NOTE: only call mkpath when not mounted (avoid hang + # when cephfs is offline + + mkpath $path if !(defined($scfg->{mkdir}) && !$scfg->{mkdir}); + + die "unable to activate storage '$storeid' - " . + "directory '$path' does not exist\n" if ! -d $path; + + cephfs_mount($scfg, $storeid); + } + + $class->SUPER::activate_storage($storeid, $scfg, $cache); +} + +sub deactivate_storage { + my ($class, $storeid, $scfg, $cache) = @_; + + $cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts() + if !$cache->{mountdata}; + + my $path = $scfg->{path}; + + if (cephfs_is_mounted($scfg, $storeid, $cache->{mountdata})) { + my $cmd = ['/bin/umount', $path]; + run_command($cmd, errmsg => 'umount error'); + } +} + +1; diff --git a/PVE/Storage/CephTools.pm b/PVE/Storage/CephTools.pm index 3e2cede..766163d 100644 --- a/PVE/Storage/CephTools.pm +++ b/PVE/Storage/CephTools.pm @@ -34,6 +34,64 @@ my $ceph_check_keyfile = sub { return undef; }; +my $parse_ceph_file = sub { + my ($filename) = @_; + + my $cfg = {}; + + return $cfg if ! -f $filename; + + my $content = PVE::Tools::file_get_contents($filename); + my @lines = split /\n/, $content; + + my $section; + + foreach my $line (@lines) { + $line =~ s/[;#].*$//; + $line =~ s/^\s+//; + $line =~ s/\s+$//; + next if !$line; + + $section = $1 if $line =~ m/^\[(\S+)\]$/; + if (!$section) { + warn "no section - skip: $line\n"; + next; + } + + if ($line =~ m/^(.*?\S)\s*=\s*(\S.*)$/) { + $cfg->{$section}->{$1} = $2; + } + + } + + return $cfg; +}; + +my $ceph_get_key = sub { + my ($keyfile, $username) = @_; + + my $key = $parse_ceph_file->($keyfile); + my $secret = $key->{"client.$username"}->{key}; + + return $secret; +}; + +sub get_monaddr_list { + my ($configfile) = shift; + + my $server; + + if (!defined($configfile)) { + warn "No ceph config specified\n"; + return; + } + + my $config = $parse_ceph_file->($configfile); + @$server = sort map { $config->{$_}->{'mon addr'} } grep {/mon/} %{$config}; + + return join(',', @$server); +}; + sub hostlist { my ($list_text, $separator) = @_; @@ -85,4 +143,48 @@ sub ceph_connect_option { } +sub ceph_create_keyfile { + my ($type, $storeid) = @_; + + my $extension = 'keyring'; + $extension = 'secret' if ($type eq 'cephfs'); + + my $ceph_admin_keyring = '/etc/pve/priv/ceph.client.admin.keyring'; + my $ceph_storage_keyring = "/etc/pve/priv/ceph/${storeid}.$extension"; + + die "ceph authx keyring file for storage '$storeid' already exists!\n" + if -e $ceph_storage_keyring; + + if (-e $ceph_admin_keyring) { + eval { + if ($type eq 'rbd') { + mkdir '/etc/pve/priv/ceph'; + PVE::Tools::file_copy($ceph_admin_keyring, $ceph_storage_keyring); + } elsif ($type eq 'cephfs') { + my $secret = $ceph_get_key->($ceph_admin_keyring, 'admin'); + mkdir '/etc/pve/priv/ceph'; + PVE::Tools::file_set_contents($ceph_storage_keyring, $secret, 0400); + } + }; + if (my $err = $@) { + unlink $ceph_storage_keyring; + die "failed to copy ceph authx $extension for storage '$storeid': $err\n"; + } + } else { + warn "$ceph_admin_keyring not found, authentication is disabled.\n"; + } +} + +sub ceph_remove_keyfile { + my ($type, $storeid) = @_; + + my $extension = 'keyring'; + $extension = 'secret' if ($type eq 'cephfs'); + my $ceph_storage_keyring = "/etc/pve/priv/ceph/${storeid}.$extension"; + + if (-f $ceph_storage_keyring) { + unlink($ceph_storage_keyring) or warn "removing keyring of storage failed: $!\n"; + } +} + 1; diff --git a/PVE/Storage/Makefile b/PVE/Storage/Makefile index 82dadd6..c7f423f 100644 --- a/PVE/Storage/Makefile +++ b/PVE/Storage/Makefile @@ -1,4 +1,4 @@ -SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm CIFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm CephTools.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm GlusterfsPlugin.pm ZFSPoolPlugin.pm ZFSPlugin.pm DRBDPlugin.pm LvmThinPlugin.pm +SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm CIFSPlugin.pm ISCSIPlugin.pm CephFSPlugin.pm RBDPlugin.pm CephTools.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm GlusterfsPlugin.pm ZFSPoolPlugin.pm ZFSPlugin.pm DRBDPlugin.pm LvmThinPlugin.pm .PHONY: install install: diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm index 88faa47..43f3bdc 100644 --- a/PVE/Storage/Plugin.pm +++ b/PVE/Storage/Plugin.pm @@ -24,6 +24,7 @@ our @SHARED_STORAGE = ( 'nfs', 'cifs', 'rbd', + 'cephfs', 'sheepdog', 'iscsidirect', 'glusterfs', diff --git a/debian/control b/debian/control index 099b68f..a713731 100644 --- a/debian/control +++ b/debian/control @@ -28,6 +28,7 @@ Depends: cstream, udev, smbclient, ceph-common, + ceph-fuse, cifs-utils, ${perl:Depends}, Description: Proxmox VE storage management library