From f9a1bc6c9b466e6f980ba7810a50bf22408df24c Mon Sep 17 00:00:00 2001 From: Matthew Penner Date: Fri, 4 Dec 2020 09:48:47 -0700 Subject: [PATCH 1/4] Show installing status instead of offline when a server is installing --- .../Servers/Network/NewAllocationRequest.php | 3 --- .../components/server/ServerDetailsBlock.tsx | 22 ++++++++++++++++--- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/app/Http/Requests/Api/Client/Servers/Network/NewAllocationRequest.php b/app/Http/Requests/Api/Client/Servers/Network/NewAllocationRequest.php index 7628afaaf..4a29378dd 100644 --- a/app/Http/Requests/Api/Client/Servers/Network/NewAllocationRequest.php +++ b/app/Http/Requests/Api/Client/Servers/Network/NewAllocationRequest.php @@ -2,8 +2,6 @@ namespace Pterodactyl\Http\Requests\Api\Client\Servers\Network; -use Illuminate\Support\Collection; -use Pterodactyl\Models\Allocation; use Pterodactyl\Models\Permission; use Pterodactyl\Http\Requests\Api\Client\ClientApiRequest; @@ -16,5 +14,4 @@ class NewAllocationRequest extends ClientApiRequest { return Permission::ACTION_ALLOCATION_CREATE; } - } diff --git a/resources/scripts/components/server/ServerDetailsBlock.tsx b/resources/scripts/components/server/ServerDetailsBlock.tsx index 321c01943..0d87089c1 100644 --- a/resources/scripts/components/server/ServerDetailsBlock.tsx +++ b/resources/scripts/components/server/ServerDetailsBlock.tsx @@ -1,5 +1,5 @@ import React, { useEffect, useState } from 'react'; -import tw from 'twin.macro'; +import tw, { TwStyle } from 'twin.macro'; import { faCircle, faEthernet, faHdd, faMemory, faMicrochip, faServer } from '@fortawesome/free-solid-svg-icons'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; import { bytesToHuman, megabytesToHuman } from '@/helpers'; @@ -13,6 +13,21 @@ interface Stats { disk: number; } +function statusToColor (status: string|null, installing: boolean): TwStyle { + if (installing) { + status = ''; + } + + switch (status) { + case 'offline': + return tw`text-red-500`; + case 'running': + return tw`text-green-500`; + default: + return tw`text-yellow-500`; + } +} + const ServerDetailsBlock = () => { const [ stats, setStats ] = useState({ memory: 0, cpu: 0, disk: 0 }); @@ -49,6 +64,7 @@ const ServerDetailsBlock = () => { }, [ instance, connected ]); const name = ServerContext.useStoreState(state => state.server.data!.name); + const isInstalling = ServerContext.useStoreState(state => state.server.data!.isInstalling); const limits = ServerContext.useStoreState(state => state.server.data!.limits); const primaryAllocation = ServerContext.useStoreState(state => state.server.data!.allocations.filter(alloc => alloc.isDefault).map( allocation => (allocation.alias || allocation.ip) + ':' + allocation.port @@ -65,10 +81,10 @@ const ServerDetailsBlock = () => { fixedWidth css={[ tw`mr-1`, - status === 'offline' ? tw`text-red-500` : (status === 'running' ? tw`text-green-500` : tw`text-yellow-500`), + statusToColor(status, isInstalling), ]} /> -  {!status ? 'Connecting...' : status} +  {!status ? 'Connecting...' : (isInstalling ? 'Installing' : status)}

From a5cebd6bcfccb87bee272ef1b102451a128698a0 Mon Sep 17 00:00:00 2001 From: Matthew Penner Date: Sun, 6 Dec 2020 13:53:55 -0700 Subject: [PATCH 2/4] s3 backups: handle CompleteMultipartUpload and AbortMultipartUpload on the panel instead of in wings, add BACKUP_PRESIGNED_URL_LIFESPAN environment variable --- app/Extensions/Backups/BackupManager.php | 3 +- .../Backups/BackupRemoteUploadController.php | 22 ++++++---- .../Remote/Backups/BackupStatusController.php | 43 ++++++++++++++++++- .../Backups/InitiateBackupService.php | 6 ++- config/backups.php | 4 ++ 5 files changed, 65 insertions(+), 13 deletions(-) diff --git a/app/Extensions/Backups/BackupManager.php b/app/Extensions/Backups/BackupManager.php index fda52f90a..fb0e0a26a 100644 --- a/app/Extensions/Backups/BackupManager.php +++ b/app/Extensions/Backups/BackupManager.php @@ -9,6 +9,7 @@ use Illuminate\Support\Str; use Webmozart\Assert\Assert; use InvalidArgumentException; use League\Flysystem\AdapterInterface; +use Illuminate\Foundation\Application; use League\Flysystem\AwsS3v3\AwsS3Adapter; use League\Flysystem\Memory\MemoryAdapter; use Illuminate\Contracts\Config\Repository; @@ -44,7 +45,7 @@ class BackupManager * * @param \Illuminate\Foundation\Application $app */ - public function __construct($app) + public function __construct(Application $app) { $this->app = $app; $this->config = $app->make(Repository::class); diff --git a/app/Http/Controllers/Api/Remote/Backups/BackupRemoteUploadController.php b/app/Http/Controllers/Api/Remote/Backups/BackupRemoteUploadController.php index cf52fc450..4bf2f8c09 100644 --- a/app/Http/Controllers/Api/Remote/Backups/BackupRemoteUploadController.php +++ b/app/Http/Controllers/Api/Remote/Backups/BackupRemoteUploadController.php @@ -6,7 +6,9 @@ use Carbon\CarbonImmutable; use Illuminate\Http\Request; use Pterodactyl\Models\Backup; use Illuminate\Http\JsonResponse; +use Illuminate\Foundation\Application; use League\Flysystem\AwsS3v3\AwsS3Adapter; +use Illuminate\Contracts\Config\Repository; use Pterodactyl\Http\Controllers\Controller; use Pterodactyl\Extensions\Backups\BackupManager; use Pterodactyl\Repositories\Eloquent\BackupRepository; @@ -16,6 +18,11 @@ class BackupRemoteUploadController extends Controller { const PART_SIZE = 5 * 1024 * 1024 * 1024; + /** + * @var \Illuminate\Contracts\Config\Repository + */ + protected $config; + /** * @var \Pterodactyl\Repositories\Eloquent\BackupRepository */ @@ -29,11 +36,13 @@ class BackupRemoteUploadController extends Controller /** * BackupRemoteUploadController constructor. * + * @param \Illuminate\Foundation\Application $app * @param \Pterodactyl\Repositories\Eloquent\BackupRepository $repository * @param \Pterodactyl\Extensions\Backups\BackupManager $backupManager */ - public function __construct(BackupRepository $repository, BackupManager $backupManager) + public function __construct(Application $app, BackupRepository $repository, BackupManager $backupManager) { + $this->config = $app->make(Repository::class); $this->repository = $repository; $this->backupManager = $backupManager; } @@ -69,7 +78,7 @@ class BackupRemoteUploadController extends Controller // Ensure we are using the S3 adapter. $adapter = $this->backupManager->adapter(); if (! $adapter instanceof AwsS3Adapter) { - throw new BadRequestHttpException('The configured backup adapter is not an S3 compatiable adapter.'); + throw new BadRequestHttpException('The configured backup adapter is not an S3 compatible adapter.'); } // The path where backup will be uploaded to @@ -77,7 +86,7 @@ class BackupRemoteUploadController extends Controller // Get the S3 client $client = $adapter->getClient(); - $expires = CarbonImmutable::now()->addMinutes(30); + $expires = CarbonImmutable::now()->addMinutes($this->config->get('backups.presigned_url_lifespan', 60)); // Params for generating the presigned urls $params = [ @@ -102,14 +111,9 @@ class BackupRemoteUploadController extends Controller } return new JsonResponse([ + 'upload_id' => $params['UploadId'], 'parts' => $parts, 'part_size' => self::PART_SIZE, - 'complete_multipart_upload' => $client->createPresignedRequest( - $client->getCommand('CompleteMultipartUpload', $params), $expires - )->getUri()->__toString(), - 'abort_multipart_upload' => $client->createPresignedRequest( - $client->getCommand('AbortMultipartUpload', $params), $expires->addMinutes(15) - )->getUri()->__toString(), ]); } } diff --git a/app/Http/Controllers/Api/Remote/Backups/BackupStatusController.php b/app/Http/Controllers/Api/Remote/Backups/BackupStatusController.php index 85a3acf6a..dac94f8d5 100644 --- a/app/Http/Controllers/Api/Remote/Backups/BackupStatusController.php +++ b/app/Http/Controllers/Api/Remote/Backups/BackupStatusController.php @@ -3,8 +3,11 @@ namespace Pterodactyl\Http\Controllers\Api\Remote\Backups; use Carbon\CarbonImmutable; +use Pterodactyl\Models\Backup; use Illuminate\Http\JsonResponse; +use League\Flysystem\AwsS3v3\AwsS3Adapter; use Pterodactyl\Http\Controllers\Controller; +use Pterodactyl\Extensions\Backups\BackupManager; use Pterodactyl\Repositories\Eloquent\BackupRepository; use Symfony\Component\HttpKernel\Exception\BadRequestHttpException; use Pterodactyl\Http\Requests\Api\Remote\ReportBackupCompleteRequest; @@ -16,14 +19,21 @@ class BackupStatusController extends Controller */ private $repository; + /** + * @var \Pterodactyl\Extensions\Backups\BackupManager + */ + private $backupManager; + /** * BackupStatusController constructor. * * @param \Pterodactyl\Repositories\Eloquent\BackupRepository $repository + * @param \Pterodactyl\Extensions\Backups\BackupManager $backupManager */ - public function __construct(BackupRepository $repository) + public function __construct(BackupRepository $repository, BackupManager $backupManager) { $this->repository = $repository; + $this->backupManager = $backupManager; } /** @@ -31,9 +41,11 @@ class BackupStatusController extends Controller * * @param \Pterodactyl\Http\Requests\Api\Remote\ReportBackupCompleteRequest $request * @param string $backup + * * @return \Illuminate\Http\JsonResponse * * @throws \Pterodactyl\Exceptions\Repository\RecordNotFoundException + * @throws \Exception */ public function __invoke(ReportBackupCompleteRequest $request, string $backup) { @@ -47,6 +59,8 @@ class BackupStatusController extends Controller } $successful = $request->input('successful') ? true : false; + + // TODO: Still run s3 code even if this fails. $model->forceFill([ 'is_successful' => $successful, 'checksum' => $successful ? ($request->input('checksum_type') . ':' . $request->input('checksum')) : null, @@ -54,6 +68,33 @@ class BackupStatusController extends Controller 'completed_at' => CarbonImmutable::now(), ])->save(); + // Check if we are using the s3 backup adapter. + $adapter = $this->backupManager->adapter(); + if ($adapter instanceof AwsS3Adapter) { + /** @var \Pterodactyl\Models\Backup $backup */ + $backup = Backup::query()->where('uuid', $backup)->firstOrFail(); + + $client = $adapter->getClient(); + + $params = [ + 'Bucket' => $adapter->getBucket(), + 'Key' => sprintf('%s/%s.tar.gz', $backup->server->uuid, $backup->uuid), + 'UploadId' => $request->input('upload_id'), + ]; + + // If the backup was not successful, send an AbortMultipartUpload request. + if (! $successful) { + $client->execute($client->getCommand('AbortMultipartUpload', $params)); + return new JsonResponse([], JsonResponse::HTTP_NO_CONTENT); + } + + // Otherwise send a CompleteMultipartUpload request. + $params['MultipartUpload'] = [ + 'Parts' => $client->execute($client->getCommand('ListParts', $params))['Parts'], + ]; + $client->execute($client->getCommand('CompleteMultipartUpload', $params)); + } + return new JsonResponse([], JsonResponse::HTTP_NO_CONTENT); } } diff --git a/app/Services/Backups/InitiateBackupService.php b/app/Services/Backups/InitiateBackupService.php index 5c34961a4..c7b884537 100644 --- a/app/Services/Backups/InitiateBackupService.php +++ b/app/Services/Backups/InitiateBackupService.php @@ -99,6 +99,8 @@ class InitiateBackupService * * @param \Pterodactyl\Models\Server $server * @param string|null $name + * @param bool $override + * * @return \Pterodactyl\Models\Backup * * @throws \Throwable @@ -107,13 +109,13 @@ class InitiateBackupService */ public function handle(Server $server, string $name = null, bool $override = false): Backup { - $previous = $this->repository->getBackupsGeneratedDuringTimespan($server->id, 10); + /*$previous = $this->repository->getBackupsGeneratedDuringTimespan($server->id, 10); if ($previous->count() >= 2) { throw new TooManyRequestsHttpException( CarbonImmutable::now()->diffInSeconds($previous->last()->created_at->addMinutes(10)), 'Only two backups may be generated within a 10 minute span of time.' ); - } + }*/ // Check if the server has reached or exceeded it's backup limit if (!$server->backup_limit || $server->backups()->where('is_successful', true)->count() >= $server->backup_limit) { diff --git a/config/backups.php b/config/backups.php index ec9bd175e..32ee1aa8a 100644 --- a/config/backups.php +++ b/config/backups.php @@ -8,6 +8,10 @@ return [ // have been made, without losing data. 'default' => env('APP_BACKUP_DRIVER', Backup::ADAPTER_WINGS), + // This value is used to determine the lifespan of UploadPart presigned urls that wings + // uses to upload backups to S3 storage. Value is in minutes, so this would default to an hour. + 'presigned_url_lifespan' => env('BACKUP_PRESIGNED_URL_LIFESPAN', 60), + 'disks' => [ // There is no configuration for the local disk for Wings. That configuration // is determined by the Daemon configuration, and not the Panel. From 1ce6d3bbba8cc844a83a8ca47ca327f45e80bca2 Mon Sep 17 00:00:00 2001 From: Matthew Penner Date: Sun, 6 Dec 2020 13:55:45 -0700 Subject: [PATCH 3/4] Maybe we should keep that backup rate-limit --- app/Services/Backups/InitiateBackupService.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/Services/Backups/InitiateBackupService.php b/app/Services/Backups/InitiateBackupService.php index c7b884537..210e92342 100644 --- a/app/Services/Backups/InitiateBackupService.php +++ b/app/Services/Backups/InitiateBackupService.php @@ -109,13 +109,13 @@ class InitiateBackupService */ public function handle(Server $server, string $name = null, bool $override = false): Backup { - /*$previous = $this->repository->getBackupsGeneratedDuringTimespan($server->id, 10); + $previous = $this->repository->getBackupsGeneratedDuringTimespan($server->id, 10); if ($previous->count() >= 2) { throw new TooManyRequestsHttpException( CarbonImmutable::now()->diffInSeconds($previous->last()->created_at->addMinutes(10)), 'Only two backups may be generated within a 10 minute span of time.' ); - }*/ + } // Check if the server has reached or exceeded it's backup limit if (!$server->backup_limit || $server->backups()->where('is_successful', true)->count() >= $server->backup_limit) { From 00429c39114e628d06fb9df1199386e2a25717f4 Mon Sep 17 00:00:00 2001 From: Matthew Penner Date: Sun, 6 Dec 2020 15:19:54 -0700 Subject: [PATCH 4/4] Cleanup remote backup controllers --- .../Backups/BackupRemoteUploadController.php | 13 ++----------- .../Api/Remote/Backups/BackupStatusController.php | 14 ++++++-------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/app/Http/Controllers/Api/Remote/Backups/BackupRemoteUploadController.php b/app/Http/Controllers/Api/Remote/Backups/BackupRemoteUploadController.php index 4bf2f8c09..497d12904 100644 --- a/app/Http/Controllers/Api/Remote/Backups/BackupRemoteUploadController.php +++ b/app/Http/Controllers/Api/Remote/Backups/BackupRemoteUploadController.php @@ -6,9 +6,7 @@ use Carbon\CarbonImmutable; use Illuminate\Http\Request; use Pterodactyl\Models\Backup; use Illuminate\Http\JsonResponse; -use Illuminate\Foundation\Application; use League\Flysystem\AwsS3v3\AwsS3Adapter; -use Illuminate\Contracts\Config\Repository; use Pterodactyl\Http\Controllers\Controller; use Pterodactyl\Extensions\Backups\BackupManager; use Pterodactyl\Repositories\Eloquent\BackupRepository; @@ -18,11 +16,6 @@ class BackupRemoteUploadController extends Controller { const PART_SIZE = 5 * 1024 * 1024 * 1024; - /** - * @var \Illuminate\Contracts\Config\Repository - */ - protected $config; - /** * @var \Pterodactyl\Repositories\Eloquent\BackupRepository */ @@ -36,13 +29,11 @@ class BackupRemoteUploadController extends Controller /** * BackupRemoteUploadController constructor. * - * @param \Illuminate\Foundation\Application $app * @param \Pterodactyl\Repositories\Eloquent\BackupRepository $repository * @param \Pterodactyl\Extensions\Backups\BackupManager $backupManager */ - public function __construct(Application $app, BackupRepository $repository, BackupManager $backupManager) + public function __construct(BackupRepository $repository, BackupManager $backupManager) { - $this->config = $app->make(Repository::class); $this->repository = $repository; $this->backupManager = $backupManager; } @@ -86,7 +77,7 @@ class BackupRemoteUploadController extends Controller // Get the S3 client $client = $adapter->getClient(); - $expires = CarbonImmutable::now()->addMinutes($this->config->get('backups.presigned_url_lifespan', 60)); + $expires = CarbonImmutable::now()->addMinutes(config('backups.presigned_url_lifespan', 60)); // Params for generating the presigned urls $params = [ diff --git a/app/Http/Controllers/Api/Remote/Backups/BackupStatusController.php b/app/Http/Controllers/Api/Remote/Backups/BackupStatusController.php index dac94f8d5..ba98b56f9 100644 --- a/app/Http/Controllers/Api/Remote/Backups/BackupStatusController.php +++ b/app/Http/Controllers/Api/Remote/Backups/BackupStatusController.php @@ -60,7 +60,6 @@ class BackupStatusController extends Controller $successful = $request->input('successful') ? true : false; - // TODO: Still run s3 code even if this fails. $model->forceFill([ 'is_successful' => $successful, 'checksum' => $successful ? ($request->input('checksum_type') . ':' . $request->input('checksum')) : null, @@ -85,14 +84,13 @@ class BackupStatusController extends Controller // If the backup was not successful, send an AbortMultipartUpload request. if (! $successful) { $client->execute($client->getCommand('AbortMultipartUpload', $params)); - return new JsonResponse([], JsonResponse::HTTP_NO_CONTENT); + } else { + // Otherwise send a CompleteMultipartUpload request. + $params['MultipartUpload'] = [ + 'Parts' => $client->execute($client->getCommand('ListParts', $params))['Parts'], + ]; + $client->execute($client->getCommand('CompleteMultipartUpload', $params)); } - - // Otherwise send a CompleteMultipartUpload request. - $params['MultipartUpload'] = [ - 'Parts' => $client->execute($client->getCommand('ListParts', $params))['Parts'], - ]; - $client->execute($client->getCommand('CompleteMultipartUpload', $params)); } return new JsonResponse([], JsonResponse::HTTP_NO_CONTENT);