diff --git a/README.md b/README.md new file mode 100644 index 0000000..660ae73 --- /dev/null +++ b/README.md @@ -0,0 +1,31 @@ +Yii2 Proxy Httpclient +===================== +Native httpclient with custom or autoparsed free proxy servers + +Installation +------------ + +The preferred way to install this extension is through [composer](http://getcomposer.org/download/). + +Either run + +``` +php composer.phar require --prefer-dist shintio/yii2-proxy-httpclient "*" +``` + +or add + +``` +"shintio/yii2-proxy-httpclient": "*" +``` + +to the require section of your `composer.json` file. + + +Usage +----- + +Once the extension is installed, simply use it in your code by : + +```php +``` \ No newline at end of file diff --git a/composer.json b/composer.json new file mode 100644 index 0000000..eceab36 --- /dev/null +++ b/composer.json @@ -0,0 +1,29 @@ +{ + "name": "shintio/yii2-proxy-httpclient", + "description": "Native httpclient with custom or autoparsed free proxy servers", + "type": "yii2-extension", + "keywords": [ + "yii2", + "extension", + "proxy", + "curl", + "httpclient" + ], + "license": "BSD-3-Clause", + "authors": [ + { + "name": "Aleksey Fedorenko", + "email": "shintio.afedorenko@gmail.com" + } + ], + "require": { + "yiisoft/yii2": "~2.0.0", + "yiisoft/yii2-httpclient": "^2.0", + "electrolinux/phpquery": "^0.9.6" + }, + "autoload": { + "psr-4": { + "shintio\\yii2\\proxy\\": "src/" + } + } +} diff --git a/src/components/Client.php b/src/components/Client.php new file mode 100644 index 0000000..438acf3 --- /dev/null +++ b/src/components/Client.php @@ -0,0 +1,46 @@ +requestConfig, 'class', $this->requestConfig['class'] ?? Request::class); + + $request = parent::createRequest(); + + $proxy = $this->getProxyManager()->getProxyString($checkProxy, $checkTimeout); + + if (!$proxy && $rescan) { + $this->getProxyManager()->parseNewProxy(); + + $proxy = $this->getProxyManager()->getProxyString($checkProxy, $checkTimeout); + } + + if ($proxy) { + $request->setOptions(['proxy' => $proxy]); + } + + return $request; + } + + public function getProxyManager() + { + return $this->proxyManager ?? new ProxyManager(); + } +} \ No newline at end of file diff --git a/src/components/ProxyManager.php b/src/components/ProxyManager.php new file mode 100644 index 0000000..b81baea --- /dev/null +++ b/src/components/ProxyManager.php @@ -0,0 +1,187 @@ + 'https://free-proxy-list.net/', + 'container' => 'table#proxylisttable', + 'ip' => 0, + 'port' => 1, + 'country' => 3, + ] + ]; + + /** + * @return integer + */ + public function parseNewProxy() + { + $count = 0; + + foreach ($this->proxyLists as $proxyList) { + $url = ArrayHelper::remove($proxyList, 'url'); + $container = ArrayHelper::remove($proxyList, 'container'); + + $client = $this->getClient(); + + $request = $client->createRequest()->setMethod('get')->setUrl($url); + + $response = $request->send(); + + if ($response->isOk) { + $document = \phpQuery::newDocumentHTML($response->content); + + $proxies = []; + + foreach ($proxyList as $key => $attribute) { + foreach ($document->find("$container tr") as $item) { + /** @var \DOMElement $item */ + if ($item->childNodes[$proxyList['ip']]->tagName == 'td') { + ($this->proxyServerModel)::saveNew($item->childNodes[$proxyList['ip']]->nodeValue, + $item->childNodes[$proxyList['port']]->nodeValue, + $item->childNodes[$proxyList['password']]->nodeValue ?? null, + $item->childNodes[$proxyList['country']]->nodeValue ?? null); + + $count++; + } + } + } + } + } + + return $count; + } + + /** + * @param ProxyServerInterface|null $proxy + * @param integer $timeout + * + * @return ProxyServerInterface|boolean + */ + public function checkProxy($proxy = null, $timeout = 5) + { + if (isset($proxy)) { + return $this->sendCheckRequest($proxy, $timeout); + } else { + do { + $proxy = ($this->proxyServerModel)::findNew(); + + if (isset($proxy)) { + $proxy = $this->sendCheckRequest($proxy, $timeout); + } else { + return false; + } + } while (empty($proxy)); + + return $proxy; + } + } + + public function getClient() + { + return $this->client ?? new \yii\httpclient\Client(); + } + + /** + * @param bool $check + * @param integer $timeout + * + * @return ProxyServerInterface + */ + public function getProxy($check = true, $timeout = 5) + { + $proxy = ($this->proxyServerModel)::findActive(); + + return $check ? $this->checkProxy($proxy, $timeout) : $proxy; + } + + /** + * @param bool $check + * @param integer $timeout + * + * @return string|boolean + */ + public function getProxyString($check = true, $timeout = 5) + { + $proxy = $this->getProxy($check, $timeout); + + return $proxy ? $this->formatProxyString($proxy) : false; + } + + /** + * @param ProxyServerInterface $proxy + * + * @return string + */ + public function formatProxyString($proxy) + { + return 'tcp://' . $proxy->getIp() . ':' . $proxy->getPort(); + } + + /** + * @param ProxyServerInterface $proxy + * @param integer $timeout + * + * @return ProxyServerInterface|boolean + */ + protected function sendCheckRequest($proxy, $timeout = 5) + { + $client = $this->getClient(); + + $request = $client->createRequest()->setMethod('get')->setOptions([ + 'proxy' => $this->formatProxyString($proxy), + 'timeout' => $timeout + ])->setUrl($this->checkerUrl); + + try { + $response = $request->send(); + + if ($response->isOk) { + $ip = $this->getCheckedIp($response); + + if ($ip == $proxy->getIp()) { + $proxy->setActive(); + + return $proxy; + } + } + } catch (\Exception $exception) { + + } + + $proxy->setUnActive(); + + return false; + } + + /** + * @param Response $response + * + * @return string + */ + protected function getCheckedIp($response) + { + $json = $response->data; + + return $json['ip']; + } +} \ No newline at end of file diff --git a/src/components/Request.php b/src/components/Request.php new file mode 100644 index 0000000..579bf09 --- /dev/null +++ b/src/components/Request.php @@ -0,0 +1,7 @@ +createTable('proxy_server', [ + 'id' => $this->primaryKey(), + 'ip' => $this->string()->notNull(), + 'port' => $this->string()->notNull(), + 'password' => $this->string(), + 'country' => $this->string(), + 'status' => $this->integer()->defaultValue(0), + 'created_at' => $this->integer(), + 'updated_at' => $this->integer(), + ], $this->tableOptions()); + } + + /** + * @inheritdoc + */ + public function down() + { + $this->dropTable('proxy_server'); + } +} diff --git a/src/models/ProxyServer.php b/src/models/ProxyServer.php new file mode 100644 index 0000000..4e6e901 --- /dev/null +++ b/src/models/ProxyServer.php @@ -0,0 +1,90 @@ + TimestampBehavior::class]; + } + + /** + * {@inheritdoc} + */ + public static function tableName() + { + return '{{proxy_server}}'; + } + + /** + * {@inheritdoc} + */ + public function rules() + { + return [ + [ + [ + 'ip', + 'port' + ], + 'required' + ], + [ + [ + 'status', + 'created_at', + 'updated_at' + ], + 'integer' + ], + [ + [ + 'ip', + 'port', + 'password', + 'country' + ], + 'string', + 'max' => 255 + ], + ]; + } + + /** + * {@inheritdoc} + */ + public function attributeLabels() + { + return [ + 'id' => Yii::t('app', 'ID'), + 'ip' => Yii::t('app', 'Ip'), + 'port' => Yii::t('app', 'Port'), + 'password' => Yii::t('app', 'Password'), + 'country' => Yii::t('app', 'Country'), + 'status' => Yii::t('app', 'Status'), + 'created_at' => Yii::t('app', 'Created At'), + 'updated_at' => Yii::t('app', 'Updated At'), + ]; + } +} diff --git a/src/models/ProxyServerInterface.php b/src/models/ProxyServerInterface.php new file mode 100644 index 0000000..3fd9802 --- /dev/null +++ b/src/models/ProxyServerInterface.php @@ -0,0 +1,56 @@ +ip; + } + + public function getPort() + { + return $this->port; + } + + public function getPassword() + { + return $this->password; + } + + public function getStatus() + { + return $this->status; + } + + public static function findNew() + { + return self::findOne(['status' => 0]); + } + + public static function findActive() + { + return self::findOne(['status' => 10]); + } + + public function setActive() + { + $this->status = 10; + + $this->save(); + } + + public function setUnActive() + { + $this->status = 5; + + $this->save(); + } + + public static function saveNew($ip, $port, $password = null, $country = null) + { + $proxyServer = new self([ + 'ip' => $ip, + 'port' => $port, + 'password' => $password, + 'country' => $country, + ]); + + $proxyServer->save(); + + return $proxyServer; + } +} \ No newline at end of file