change schema

fozbek · May 25, 2021 · d7477d8 · d7477d8
1 parent 75d3cf7
commit d7477d8
Show file tree

Hide file tree

Showing 12 changed files with 3,297 additions and 254 deletions.
diff --git a/README.md b/README.md
@@ -16,7 +16,7 @@ $url = 'https://google.com';
 $schema = [
     'title' => 'title',
     'a-tags' => [
-        'selector' => 'a',
+        'list-selector' => 'a',
         'content' => [
             'text' => 'a',
             'url' => 'a@href',
@@ -47,9 +47,9 @@ Response (Formatted)
     } 
 
 #### Examples
->You can test all of these in any xenforo forum. Example url: https://xenforo.com/community/forums/announcements/
+>You can test all of these in any site that uses xenforo. Example url: https://xenforo.com/community/forums/announcements/
 
-- Scrape single selector
+- Single selector
 ```php
 $schema = [
     'forum-title' => '.p-body-header .p-title-value' 
@@ -60,7 +60,7 @@ $schema = [
 ```php
 $schema = [
     'threads' => [
-        'selector' => '.structItem--thread',
+        'list-selector' => '.structItem--thread',
         'content' => [
             'thread-title' => '.structItem-title',
             'thread-url' => '.structItem-title a@href',
@@ -70,22 +70,12 @@ $schema = [
 ];
 ``` 
 
-- Pagination
-```php
-$schema = [
-    'title' => 'title',
-    'pagination' => [
-        'limit' => 3,
-        'selector' => '.pageNav-jump--next@href',
-    ],
-];
-```
-
 - New Request
 ```php
 $schema = [
     'login-page' => [
-        'selector' => 'a.p-navgroup-link--logIn@href',
+        'request-selector' => 'a.p-navgroup-link--logIn@href',
+        'base-url' => 'https://xenforo.com',
         'content' => [
             'title' => 'title',
         ],
@@ -98,19 +88,16 @@ $schema = [
 $schema = [
     'title' => 'title',
     'threads' => [
-        'selector' => '.structItem--thread',
+        'list-selector' => '.structItem--thread',
         'content' => [
             'thread-detail' => [
-                'selector' => '.structItem-title a@href',
+                'request-selector' => '.structItem-title a@href',
+                'base-url' => 'https://xenforo.com',
                 'content' => [
                     'thread-content' => '.message-body .bbWrapper',
                 ],
             ],
         ],
-    ],
-    'pagination' => [
-        'limit' => 3,
-        'selector' => '.pageNav-jump--next@href',
-    ],
+    ]
 ];
 ```
diff --git a/composer.json b/composer.json
@@ -7,17 +7,19 @@
             "email": "mail@fatih.dev"
         }
     ],
+    "description": "Simple, schema based scraping tool",
     "autoload": {
         "psr-4": {
             "Scrawler\\": "src/"
         }
     },
     "require": {
-        "symfony/dom-crawler": "^5.0",
-        "symfony/css-selector": "^5.0",
-        "guzzlehttp/guzzle": "~6.0|^7.0.1"
+        "guzzlehttp/guzzle": "~6.0|^7.0.1",
+        "imangazaliev/didom": "^1.16",
+        "ext-dom": "*"
     },
     "require-dev": {
+        "roave/security-advisories": "dev-latest",
         "phpstan/phpstan": "^0.12.34",
         "phpunit/phpunit": "^9"
     },