From 9269776b9a05af9cbf9bde80a4f2933d11db80b7 Mon Sep 17 00:00:00 2001 From: Bilge Date: Tue, 6 Apr 2021 01:25:47 +0100 Subject: [PATCH] Added SessionResource to support session scraping in ScrapeAppDetails. WIP: SessionResource should be renamed SteamSessionResource and only async support was added thus far (TODO: sync support). This WIP was abandoned when it was discovered we do not need to log in to scrape adult games (the cookie was just renamed). --- src/Resource/Curator/CuratorResource.php | 24 ++-------------- src/Resource/ScrapeAppDetails.php | 26 +++++++++++------ src/Resource/SessionResource.php | 32 +++++++++++++++++++++ test/Functional/ScrapeAppDetailsTest.php | 36 ++++++++++++++++++------ 4 files changed, 79 insertions(+), 39 deletions(-) create mode 100644 src/Resource/SessionResource.php diff --git a/src/Resource/Curator/CuratorResource.php b/src/Resource/Curator/CuratorResource.php index 6544713..941cacf 100644 --- a/src/Resource/Curator/CuratorResource.php +++ b/src/Resource/Curator/CuratorResource.php @@ -3,35 +3,23 @@ namespace ScriptFUSION\Porter\Provider\Steam\Resource\Curator; -use Amp\Http\Client\Cookie\CookieJar; use Amp\Iterator; use Amp\Producer; -use ScriptFUSION\Porter\Connector\AsyncDataSource; use ScriptFUSION\Porter\Connector\ImportConnector; use ScriptFUSION\Porter\Net\Http\AsyncHttpConnector; use ScriptFUSION\Porter\Net\Http\HttpResponse; -use ScriptFUSION\Porter\Provider\Resource\AsyncResource; -use ScriptFUSION\Porter\Provider\Steam\SteamProvider; +use ScriptFUSION\Porter\Provider\Steam\Resource\SessionResource; -abstract class CuratorResource implements AsyncResource +abstract class CuratorResource extends SessionResource { - protected $session; - protected $curatorId; public function __construct(CuratorSession $session, int $curatorId) { - $this->session = $session; + $this->setSession($session); $this->curatorId = $curatorId; } - public function getProviderClassName(): string - { - return SteamProvider::class; - } - - abstract protected function getSource(): AsyncDataSource; - public function fetchAsync(ImportConnector $connector): Iterator { return new Producer(function (\Closure $emit) use ($connector): \Generator { @@ -58,10 +46,4 @@ protected function emitResponses(\Closure $emit, HttpResponse $response): \Gener yield $emit($json); } - - private function applySessionCookies(CookieJar $cookieJar): void - { - $cookieJar->store($this->session->getSecureLoginCookie()); - $cookieJar->store($this->session->getStoreSessionCookie()); - } } diff --git a/src/Resource/ScrapeAppDetails.php b/src/Resource/ScrapeAppDetails.php index 9fe800c..da4575d 100644 --- a/src/Resource/ScrapeAppDetails.php +++ b/src/Resource/ScrapeAppDetails.php @@ -16,24 +16,25 @@ use ScriptFUSION\Porter\Provider\Resource\AsyncResource; use ScriptFUSION\Porter\Provider\Resource\ProviderResource; use ScriptFUSION\Porter\Provider\Resource\SingleRecordResource; +use ScriptFUSION\Porter\Provider\Steam\Resource\Curator\CuratorSession; use ScriptFUSION\Porter\Provider\Steam\Scrape\AppDetailsParser; use ScriptFUSION\Porter\Provider\Steam\SteamProvider; /** * Scrapes the Steam store page for App details. */ -final class ScrapeAppDetails implements ProviderResource, SingleRecordResource, AsyncResource, Url +final class ScrapeAppDetails extends SessionResource implements + ProviderResource, + AsyncResource, + SingleRecordResource, + Url { private $appId; - public function __construct(int $appId) + public function __construct(int $appId, CuratorSession $curatorSession = null) { $this->appId = $appId; - } - - public function getProviderClassName(): string - { - return SteamProvider::class; + $curatorSession && $this->setSession($curatorSession); } public function fetch(ImportConnector $connector): \Iterator @@ -43,7 +44,7 @@ public function fetch(ImportConnector $connector): \Iterator $this->validateResponse($response = $connector->fetch( (new HttpDataSource($this->getUrl())) // Enable age-restricted and mature content. - ->addHeader('Cookie: birthtime=0; mature_content=1') + ->addHeader('Cookie: birthtime=0; wants_mature_content=1') )); yield AppDetailsParser::tryParseStorePage($response->getBody()); @@ -54,6 +55,13 @@ public function fetchAsync(ImportConnector $connector): Iterator $this->configureAsyncOptions($connector->findBaseConnector()); return new Producer(function (\Closure $emit) use ($connector): \Generator { + $baseConnector = $connector->findBaseConnector(); + if (!$baseConnector instanceof AsyncHttpConnector) { + throw new \InvalidArgumentException('Unexpected connector type.'); + } + + $this->applySessionCookies($baseConnector->getCookieJar()); + /** @var HttpResponse $response */ $this->validateResponse($response = yield $connector->fetchAsync( (new AsyncHttpDataSource($this->getUrl())) @@ -99,6 +107,6 @@ private function configureAsyncOptions(AsyncHttpConnector $connector): void // Enable age-restricted content. $cookies->store(new ResponseCookie('birthtime', '0', $cookieAttributes)); // Enable mature content. - $cookies->store(new ResponseCookie('mature_content', '1', $cookieAttributes)); + $cookies->store(new ResponseCookie('wants_mature_content', '1', $cookieAttributes)); } } diff --git a/src/Resource/SessionResource.php b/src/Resource/SessionResource.php new file mode 100644 index 0000000..ee2b0cb --- /dev/null +++ b/src/Resource/SessionResource.php @@ -0,0 +1,32 @@ +session = $curatorSession; + } + + public function getProviderClassName(): string + { + return SteamProvider::class; + } + + protected function applySessionCookies(CookieJar $cookieJar): void + { + if ($this->session) { + $cookieJar->store($this->session->getSecureLoginCookie()); + $cookieJar->store($this->session->getStoreSessionCookie()); + } + } +} diff --git a/test/Functional/ScrapeAppDetailsTest.php b/test/Functional/ScrapeAppDetailsTest.php index d79b22f..68dcf58 100644 --- a/test/Functional/ScrapeAppDetailsTest.php +++ b/test/Functional/ScrapeAppDetailsTest.php @@ -3,15 +3,8 @@ namespace ScriptFUSIONTest\Porter\Provider\Steam\Functional; -use Amp\Http\Client\Cookie\InMemoryCookieJar; -use Amp\Success; use Mockery\Adapter\Phpunit\MockeryPHPUnitIntegration; -use Mockery\Generator\MockConfigurationBuilder; use PHPUnit\Framework\TestCase; -use ScriptFUSION\Porter\Connector\ImportConnectorFactory; -use ScriptFUSION\Porter\Net\Http\AsyncHttpConnector; -use ScriptFUSION\Porter\Net\Http\AsyncHttpOptions; -use ScriptFUSION\Porter\Net\Http\HttpResponse; use ScriptFUSION\Porter\Porter; use ScriptFUSION\Porter\Provider\Steam\Resource\InvalidAppIdException; use ScriptFUSION\Porter\Provider\Steam\Resource\ScrapeAppDetails; @@ -19,10 +12,8 @@ use ScriptFUSION\Porter\Provider\Steam\Scrape\SteamStoreException; use ScriptFUSION\Porter\Specification\AsyncImportSpecification; use ScriptFUSION\Porter\Specification\ImportSpecification; -use ScriptFUSION\Retry\FailingTooHardException; use ScriptFUSIONTest\Porter\Provider\Steam\Fixture\ScrapeAppFixture; use ScriptFUSIONTest\Porter\Provider\Steam\FixtureFactory; -use function Amp\Iterator\toArray; use function Amp\Promise\wait; /** @@ -776,6 +767,33 @@ public function testValveIndex(): void self::assertNull($app['release_date']); } + /** + * Tests that an adult game behind a login wall can be accessed and parsed with a valid login. + * + * @see https://store.steampowered.com/app/1296770/Her_New_Memory__Hentai_Simulator/ + * + * @dataProvider provideAdultGameBisync + */ + public function testAdultGame(\Closure $import): void + { + $app = \Closure::bind($import, $this)(); + + self::assertSame('Her New Memory - Hentai Simulator', $app['name']); + self::assertSame('game', $app['type']); + self::assertSame(1296770, $app['app_id']); + self::assertTrue($app['windows']); + self::assertNotEmpty($app['tags']); + self::assertNotEmpty($app['languages']); + } + + /** + * Provides a game synchronously and asynchronously. + */ + public function provideAdultGameBisync(): \Generator + { + return $this->provideAppBisync(1296770); + } + /** * Tests that when non-HTML markup is returned, InvalidMarkupException is thrown. */