diff --git a/app/Extensions/Backups/BackupManager.php b/app/Extensions/Backups/BackupManager.php new file mode 100644 index 000000000..36384297d --- /dev/null +++ b/app/Extensions/Backups/BackupManager.php @@ -0,0 +1,227 @@ +app = $app; + $this->config = $app->make(Repository::class); + } + + /** + * Returns a backup adapter instance. + * + * @param string|null $name + * @return \League\Flysystem\AdapterInterface + */ + public function adapter(string $name = null) + { + return $this->get($name ?: $this->getDefaultAdapter()); + } + + /** + * Set the given backup adapter instance. + * + * @param string $name + * @param \League\Flysystem\AdapterInterface $disk + * @return $this + */ + public function set(string $name, $disk) + { + $this->adapters[$name] = $disk; + + return $this; + } + + /** + * Gets a backup adapter. + * + * @param string $name + * @return \League\Flysystem\AdapterInterface + */ + protected function get(string $name) + { + return $this->adapters[$name] = $this->resolve($name); + } + + /** + * Resolve the given backup disk. + * + * @param string $name + * @return \League\Flysystem\AdapterInterface + */ + protected function resolve(string $name) + { + $config = $this->getConfig($name); + + if (empty($config['adapter'])) { + throw new InvalidArgumentException( + "Backup disk [{$name}] does not have a configured adapter." + ); + } + + $adapter = $config['adapter']; + + if (isset($this->customCreators[$name])) { + return $this->callCustomCreator($config); + } + + $adapterMethod = 'create' . Str::studly($adapter) . 'Adapter'; + if (method_exists($this, $adapterMethod)) { + $instance = $this->{$adapterMethod}($config); + + Assert::isInstanceOf($instance, AdapterInterface::class); + + return $instance; + } + + throw new InvalidArgumentException("Adapter [{$adapter}] is not supported."); + } + + /** + * Calls a custom creator for a given adapter type. + * + * @param array $config + * @return \League\Flysystem\AdapterInterface + */ + protected function callCustomCreator(array $config) + { + $adapter = $this->customCreators[$config['adapter']]($this->app, $config); + + Assert::isInstanceOf($adapter, AdapterInterface::class); + + return $adapter; + } + + /** + * Creates a new wings adapter. + * + * @param array $config + * @return \League\Flysystem\AdapterInterface + */ + public function createWingsAdapter(array $config) + { + return new MemoryAdapter(null); + } + + /** + * Creates a new S3 adapter. + * + * @param array $config + * @return \League\Flysystem\AdapterInterface + */ + public function createS3Adapter(array $config) + { + $config['version'] = 'latest'; + + if (! empty($config['key']) && ! empty($config['secret'])) { + $config['credentials'] = Arr::only($config, ['key', 'secret', 'token']); + } + + $client = new S3MultiRegionClient($config); + + return new AwsS3Adapter($client, $config['bucket'], $config['prefix'] ?? '', $config['options'] ?? []); + } + + /** + * Returns the configuration associated with a given backup type. + * + * @param string $name + * @return array + */ + protected function getConfig(string $name) + { + return $this->config->get("backups.disks.{$name}") ?: []; + } + + /** + * Get the default backup driver name. + * + * @return string + */ + public function getDefaultAdapter() + { + return $this->config->get('backups.default'); + } + + /** + * Set the default session driver name. + * + * @param string $name + */ + public function setDefaultAdapter(string $name) + { + $this->config->set('backups.default', $name); + } + + /** + * Unset the given adapter instances. + * + * @param string|string[] $adapter + * @return $this + */ + public function forget($adapter) + { + foreach ((array) $adapter as $adapterName) { + unset($this->adapters[$adapter]); + } + + return $this; + } + + /** + * Register a custom adapter creator closure. + * + * @param string $adapter + * @param \Closure $callback + * @return $this + */ + public function extend(string $adapter, Closure $callback) + { + $this->customCreators[$adapter] = $callback; + + return $this; + } +} diff --git a/app/Models/Backup.php b/app/Models/Backup.php index d324090e5..14cb4dde9 100644 --- a/app/Models/Backup.php +++ b/app/Models/Backup.php @@ -26,8 +26,8 @@ class Backup extends Model const RESOURCE_NAME = 'backup'; - const DISK_LOCAL = 'local'; - const DISK_AWS_S3 = 's3'; + const ADAPTER_WINGS = 'wings'; + const ADAPTER_AWS_S3 = 's3'; /** * @var string diff --git a/app/Providers/BackupsServiceProvider.php b/app/Providers/BackupsServiceProvider.php new file mode 100644 index 000000000..999dfa90d --- /dev/null +++ b/app/Providers/BackupsServiceProvider.php @@ -0,0 +1,28 @@ +app->singleton(BackupManager::class, function ($app) { + return new BackupManager($app); + }); + } + + /** + * @return string[] + */ + public function provides() + { + return [BackupManager::class]; + } +} diff --git a/app/Repositories/Wings/DaemonBackupRepository.php b/app/Repositories/Wings/DaemonBackupRepository.php index e40793dd7..2e5ea89fd 100644 --- a/app/Repositories/Wings/DaemonBackupRepository.php +++ b/app/Repositories/Wings/DaemonBackupRepository.php @@ -11,25 +11,48 @@ use Pterodactyl\Exceptions\Http\Connection\DaemonConnectionException; class DaemonBackupRepository extends DaemonRepository { + /** + * @var string|null + */ + protected $adapter; + + /** + * Sets the backup adapter for this execution instance. + * + * @param string $adapter + * @return $this + */ + public function setBackupAdapter(string $adapter) + { + $this->adapter = $adapter; + + return $this; + } + /** * Tells the remote Daemon to begin generating a backup for the server. * * @param \Pterodactyl\Models\Backup $backup + * @param string|null $presignedUrl * @return \Psr\Http\Message\ResponseInterface * * @throws \Pterodactyl\Exceptions\Http\Connection\DaemonConnectionException */ - public function backup(Backup $backup): ResponseInterface + public function backup(Backup $backup, string $presignedUrl = null): ResponseInterface { Assert::isInstanceOf($this->server, Server::class); + $this->app->make('config')->get(); + try { return $this->getHttpClient()->post( sprintf('/api/servers/%s/backup', $this->server->uuid), [ 'json' => [ + 'adapter' => $this->adapter ?? config('backups.default'), 'uuid' => $backup->uuid, 'ignored_files' => $backup->ignored_files, + 'presigned_url' => $presignedUrl, ], ] ); diff --git a/app/Services/Backups/InitiateBackupService.php b/app/Services/Backups/InitiateBackupService.php index ead3bff82..fd4e53a2e 100644 --- a/app/Services/Backups/InitiateBackupService.php +++ b/app/Services/Backups/InitiateBackupService.php @@ -8,7 +8,9 @@ use Carbon\CarbonImmutable; use Webmozart\Assert\Assert; use Pterodactyl\Models\Backup; use Pterodactyl\Models\Server; +use League\Flysystem\AwsS3v3\AwsS3Adapter; use Illuminate\Database\ConnectionInterface; +use Pterodactyl\Extensions\Backups\BackupManager; use Pterodactyl\Repositories\Eloquent\BackupRepository; use Pterodactyl\Repositories\Wings\DaemonBackupRepository; use Pterodactyl\Exceptions\Service\Backup\TooManyBackupsException; @@ -36,21 +38,29 @@ class InitiateBackupService */ private $daemonBackupRepository; + /** + * @var \Pterodactyl\Extensions\Backups\BackupManager + */ + private $backupManager; + /** * InitiateBackupService constructor. * * @param \Pterodactyl\Repositories\Eloquent\BackupRepository $repository * @param \Illuminate\Database\ConnectionInterface $connection * @param \Pterodactyl\Repositories\Wings\DaemonBackupRepository $daemonBackupRepository + * @param \Pterodactyl\Extensions\Backups\BackupManager $backupManager */ public function __construct( BackupRepository $repository, ConnectionInterface $connection, - DaemonBackupRepository $daemonBackupRepository + DaemonBackupRepository $daemonBackupRepository, + BackupManager $backupManager ) { $this->repository = $repository; $this->connection = $connection; $this->daemonBackupRepository = $daemonBackupRepository; + $this->backupManager = $backupManager; } /** @@ -110,12 +120,45 @@ class InitiateBackupService 'uuid' => Uuid::uuid4()->toString(), 'name' => trim($name) ?: sprintf('Backup at %s', CarbonImmutable::now()->toDateTimeString()), 'ignored_files' => is_array($this->ignoredFiles) ? array_values($this->ignoredFiles) : [], - 'disk' => 'local', + 'disk' => $this->backupManager->getDefaultAdapter(), ], true, true); - $this->daemonBackupRepository->setServer($server)->backup($backup); + $url = $this->getS3PresignedUrl(sprintf('%s/%s.tar.gz', $server->uuid, $backup->uuid)); + + $this->daemonBackupRepository->setServer($server) + ->setBackupAdapter($this->backupManager->getDefaultAdapter()) + ->backup($backup, $url); return $backup; }); } + + /** + * Generates a presigned URL for the wings daemon to upload the completed archive + * to. We use a 30 minute expiration on these URLs to avoid issues with large backups + * that may take some time to complete. + * + * @param string $path + * @return string|null + */ + protected function getS3PresignedUrl(string $path) + { + $adapter = $this->backupManager->adapter(); + if (! $adapter instanceof AwsS3Adapter) { + return null; + } + + $client = $adapter->getClient(); + + $request = $client->createPresignedRequest( + $client->getCommand('PutObject', [ + 'Bucket' => $adapter->getBucket(), + 'Key' => $path, + 'ContentType' => 'binary/octet-stream', + ]), + CarbonImmutable::now()->addMinutes(30) + ); + + return $request->getUri()->__toString(); + } } diff --git a/composer.json b/composer.json index f582e060a..c8cd7a290 100644 --- a/composer.json +++ b/composer.json @@ -27,6 +27,8 @@ "laravel/helpers": "^1.2", "laravel/tinker": "^1.0", "lcobucci/jwt": "^3.3", + "league/flysystem-aws-s3-v3": "^1.0", + "league/flysystem-memory": "^1.0", "matriphe/iso-639": "^1.2", "pragmarx/google2fa": "^5.0", "predis/predis": "^1.1", diff --git a/composer.lock b/composer.lock index 6cc218722..0a6709686 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "1d03ca0a7151c6594255643eaaa8f527", + "content-hash": "2318244deed430b3f0b3df9ee4977759", "packages": [ { "name": "appstract/laravel-blade-directives", @@ -1663,6 +1663,104 @@ ], "time": "2020-03-17T18:58:12+00:00" }, + { + "name": "league/flysystem-aws-s3-v3", + "version": "1.0.24", + "source": { + "type": "git", + "url": "https://github.com/thephpleague/flysystem-aws-s3-v3.git", + "reference": "4382036bde5dc926f9b8b337e5bdb15e5ec7b570" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thephpleague/flysystem-aws-s3-v3/zipball/4382036bde5dc926f9b8b337e5bdb15e5ec7b570", + "reference": "4382036bde5dc926f9b8b337e5bdb15e5ec7b570", + "shasum": "" + }, + "require": { + "aws/aws-sdk-php": "^3.0.0", + "league/flysystem": "^1.0.40", + "php": ">=5.5.0" + }, + "require-dev": { + "henrikbjorn/phpspec-code-coverage": "~1.0.1", + "phpspec/phpspec": "^2.0.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0-dev" + } + }, + "autoload": { + "psr-4": { + "League\\Flysystem\\AwsS3v3\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Frank de Jonge", + "email": "info@frenky.net" + } + ], + "description": "Flysystem adapter for the AWS S3 SDK v3.x", + "time": "2020-02-23T13:31:58+00:00" + }, + { + "name": "league/flysystem-memory", + "version": "1.0.2", + "source": { + "type": "git", + "url": "https://github.com/thephpleague/flysystem-memory.git", + "reference": "d0e87477c32e29f999b4de05e64c1adcddb51757" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thephpleague/flysystem-memory/zipball/d0e87477c32e29f999b4de05e64c1adcddb51757", + "reference": "d0e87477c32e29f999b4de05e64c1adcddb51757", + "shasum": "" + }, + "require": { + "league/flysystem": "~1.0" + }, + "require-dev": { + "phpunit/phpunit": "^5.7.10" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0-dev" + } + }, + "autoload": { + "psr-4": { + "League\\Flysystem\\Memory\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Chris Leppanen", + "email": "chris.leppanen@gmail.com", + "role": "Developer" + } + ], + "description": "An in-memory adapter for Flysystem.", + "homepage": "https://github.com/thephpleague/flysystem-memory", + "keywords": [ + "Flysystem", + "adapter", + "memory" + ], + "time": "2019-05-30T21:34:13+00:00" + }, { "name": "league/fractal", "version": "0.19.2", diff --git a/config/app.php b/config/app.php index f27b66a0e..eba84ca09 100644 --- a/config/app.php +++ b/config/app.php @@ -175,6 +175,7 @@ return [ */ Pterodactyl\Providers\AppServiceProvider::class, Pterodactyl\Providers\AuthServiceProvider::class, + Pterodactyl\Providers\BackupsServiceProvider::class, Pterodactyl\Providers\BladeServiceProvider::class, Pterodactyl\Providers\EventServiceProvider::class, Pterodactyl\Providers\HashidsServiceProvider::class, diff --git a/config/backups.php b/config/backups.php index 57edfee30..57d71c604 100644 --- a/config/backups.php +++ b/config/backups.php @@ -1,29 +1,39 @@ env('APP_BACKUP_DRIVER', 'local'), + 'default' => env('APP_BACKUP_DRIVER', 'local'), 'disks' => [ // There is no configuration for the local disk for Wings. That configuration // is determined by the Daemon configuration, and not the Panel. - 'local' => [], + 'local' => [ + 'adapter' => Backup::ADAPTER_WINGS, + ], - // Configuration for storing backups in Amazon S3. + // Configuration for storing backups in Amazon S3. This uses the same credentials + // specified in filesystems.php but does include some more specific settings for + // backups, notably bucket, location, and use_accelerate_endpoint. 's3' => [ - 'region' => '', - 'access_key' => '', - 'access_secret_key' => '', + 'adapter' => Backup::ADAPTER_AWS_S3, + + 'region' => env('AWS_DEFAULT_REGION'), + 'key' => env('AWS_ACCESS_KEY_ID'), + 'secret' => env('AWS_SECRET_ACCESS_KEY'), // The S3 bucket to use for backups. - 'bucket' => '', + 'bucket' => env('AWS_BACKUPS_BUCKET'), // The location within the S3 bucket where backups will be stored. Backups // are stored within a folder using the server's UUID as the name. Each // backup for that server lives within that folder. - 'location' => '', + 'prefix' => env('AWS_BACKUPS_BUCKET') ?? '', + + 'use_accelerate_endpoint' => env('AWS_BACKUPS_USE_ACCELERATE', false), ], ], ];