Skip to content

Commit

Permalink
Merge pull request #26 from baddiservices/dev
Browse files Browse the repository at this point in the history 
Update story
  • Loading branch information
@5baddi
5baddi authored Jan 25, 2021
2 parents 6ab391f + c28571b commit 6f94e1e
Show file tree
Hide file tree
Showing 93 changed files with 44,595 additions and 17,114 deletions.
10 changes: 10 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,16 @@ I guess the easiest way will be just use residential ip with enough high delay b

You can ignore pretty much any free IP proxy list available on google, 99% of those ips on it are banned, almost same with ips from Digitalocean, OVH etc, many of them are blocked as well.

302 redirect (to login page) typically happens if you try to connect to Instagram via datacenter ip range.
Instagram only allows residential ip addresses so your best bet is to use:

1) your home network
2) clean residential proxies
3) use this [paid package](https://github.com/restyler/instagram-php-scraper)

To prevent many requests response use random delay between requests also delay by minutes very recommended.


## Documentation
* [Setup & database migration](/docs/setup.md)
* [Users hierarchy](/docs/users_hierarchy.md)
2 changes: 1 addition & 1 deletion app/Brand.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ public function getTrackersCountAttribute() : int
*/
public function getLogoAttribute() : ?string
{
if(isset($this->attributes['logo']))
if(isset($this->attributes['logo']) && Storage::disk('local')->exists($this->attributes['logo']))
return "data:image/png;base64," . base64_encode(Storage::disk('local')->get($this->attributes['logo']));

return null;
Expand Down
9 changes: 9 additions & 0 deletions app/CampaignAnalytics.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,15 @@ class CampaignAnalytics extends Model
*/
protected $table = 'campaign_analytics';

/**
* The attributes that should be cast to native types.
*
* @var array
*/
protected $casts = [
'top_emojis' => 'json',
];

/**
* Get top emojis array with keys
*
Expand Down
56 changes: 39 additions & 17 deletions app/Console/Commands/InstagramCommand.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ private function scrapInfluencers() : void
// Scrap each influencer details
$influencers->each(function($influencer){
// Ignore last updated influencers
if($influencer->posts_count === $influencer->medias){
if($influencer->posts_count === $influencer->medias || $influencer->scraped){
// Remove influnecer from process
$influencer->update(['in_process' => false]);

Expand All @@ -108,19 +108,11 @@ private function scrapInfluencers() : void
$this->info("Already scraped posts: {$influencer->posts_count}");
$this->info("Please wait until scraping all medias ...");

// Get next cursor
$lastPost = InfluencerPost::where('influencer_id', $influencer->id)
->whereNotNull('next_cursor')
->orWhere('next_cursor', '!=', '')
->orderBy('id', 'desc')
->latest()
->first();

// Scrap new medias
$this->instagramScraper->getMedias($influencer, $lastPost->next_cursor ?? null);
// Get influencer media
$this->getInfluencerMedia($influencer);

// Remove influnecer from process
$influencer->update(['in_process' => false]);
$influencer->update(['in_process' => false, 'scraped' => true]);
}catch(\Exception $exception){
// Trace
Log::error($exception->getMessage(), [
Expand Down Expand Up @@ -159,22 +151,32 @@ private function updateInfluencers() : void

// Update influencer
$influencer->update($accountDetails);
$influencer->refresh();
$this->info("Successfully updated influencer @{$influencer->username}");

// Get new media
if($influencer->posts_count < $influencer->medias)
$this->getInfluencerMedia($influencer);

// Load posts
$influencer->load('posts');
// Handle by posts
$influencer->posts->chunk(50, function($posts){
$posts->each(function($post){
// Get online media
$_media = $this->instagramScraper->byMedia($post->short_code);
$this->info("Media {$media['short_code']} successfully scraped!");
try{
// Get online media
$_media = $this->instagramScraper->byMedia($post->short_code);
$this->info("Media {$media['short_code']} successfully scraped!");
}catch(\Exception $ex){
// TODO: Handle not found media exception
$post->delete();

return true;
}

// Format media
$media = $this->instagramScraper->getMedia($_media);

// TODO: remove deleted media

// Update comments if post linked to a tracker
$mediaTrackersCount = TrackerInfluencerMedia::where('influencer_post_id', $post->id)->count();
if($mediaTrackersCount > 0){
Expand Down Expand Up @@ -202,4 +204,24 @@ private function updateInfluencers() : void
});
});
}

/**
* Get influencer media
*
* @param \App\Influencer $influencer
* @return void
*/
private function getInfluencerMedia(Influencer $influencer) : void
{
// Get next cursor
$lastPost = InfluencerPost::where('influencer_id', $influencer->id)
->whereNotNull('next_cursor')
->orWhere('next_cursor', '!=', '')
->orderBy('id', 'desc')
->latest()
->first();

// Scrap new medias
$this->instagramScraper->getMedias($influencer, $lastPost->next_cursor ?? null);
}
}
34 changes: 33 additions & 1 deletion app/Console/Commands/UpdateAnalyticsCommand.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,8 @@ private function campaignsAnalytics()
'organic_video_views' => 0,
'comments_count' => 0,
'posts_count' => 0,
'images_count' => 0,
'videos_count' => 0,
'organic_posts' => 0,
'stories_count' => 0,
'links_count' => 0,
Expand Down Expand Up @@ -130,6 +132,16 @@ private function campaignsAnalytics()
if(!is_null($tracker->analytics->organic_video_views))
$analytics['organic_posts'] += $tracker->analytics->organic_posts;

// Count posts for each type
if(!is_null($tracker->analytics->videos_count))
$analytics['videos_count'] += $tracker->analytics->videos_count;
if(!is_null($tracker->analytics->images_count))
$analytics['images_count'] += $tracker->analytics->images_count;
if($tracker->analytics->is_link)
$analytics['links_count'] += 1;
if($tracker->analytics->is_story)
$analytics['stories_count'] += 1;

// Emojis
if(isset($tracker->analytics->top_emojis['top'], $tracker->analytics->top_emojis['all'])){
$analytics['top_emojis']['top'] = array_merge($analytics['top_emojis']['top'], $tracker->analytics->top_emojis['top']);
Expand Down Expand Up @@ -193,6 +205,10 @@ private function trackersAnalytics()
'organic_video_views' => 0,
'comments_count' => 0,
'posts_count' => 0,
'images_count' => 0,
'videos_count' => 0,
'is_link' => false,
'is_story' => false,
'organic_posts' => 0,
'top_emojis' => [],
'sentiments_positive' => 0.0,
Expand All @@ -205,6 +221,12 @@ private function trackersAnalytics()
case "post":
$analytics['posts_count'] = $tracker->posts->count();
break;
case "url":
$analytics['is_link'] = true;
break;
case "story":
$analytics['is_story'] = true;
break;
}

// Calculate analytics
Expand All @@ -228,7 +250,17 @@ private function trackersAnalytics()
$analytics['organic_impressions'] += $post->likes + $post->video_views;

if($tracker->type === 'post')
$analytics['organic_posts'] = $analytics['organic_posts'] + 1;
$analytics['organic_posts'] += 1;
}

// Count posts for each type
switch($post->type){
case "video":
$analytics['videos_count'] += 1;
break;
default:
$analytics['images_count'] += 1;
break;
}

// TODO: store new values about comment owner gender
Expand Down
37 changes: 35 additions & 2 deletions app/Http/Controllers/InfluencerController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,13 @@
use App\Influencer;
use App\InfluencerPost;
use App\BrandInfluencer;
use App\InfluencerStory;
use App\Jobs\ScrapInfluencerJob;
use App\Services\YoutubeScraper;
use App\Services\InstagramScraper;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Auth;
use Illuminate\Support\Facades\Gate;
use App\Http\Resources\StoriesCollection;
use App\Http\Requests\CreateInfluencerRequest;
use Symfony\Component\HttpFoundation\Response;
use App\Http\Resources\DataTable\InfluencerDTResource;
Expand All @@ -38,7 +39,39 @@ public function byBrand(Brand $brand)
);
}

public function create(CreateInfluencerRequest $request, InstagramScraper $instagram, YoutubeScraper $youtube)
/**
* Fetch stories by brand
*
* @param \App\Brand $brand
* @return \Illuminate\Http\Response
*/
public function storiesByBrand(Brand $brand)
{
// Load brand influencers
$ids = BrandInfluencer::where('brand_id', $brand->id)->pluck('influencer_id')->toArray();

// Load stories
$stories = InfluencerStory::with(['influencer'])
->whereIn('influencer_id', $ids)
->orderBy('created_at', 'desc')
->paginate(10);

return response()->success(
"Stories fetched successfully.",
[
'items' => StoriesCollection::collection($stories->getCollection()),
'pagination' => [
'total' => $stories->total(),
'count' => $stories->count(),
'perPage' => $stories->perPage(),
'currentPage' => $stories->currentPage(),
'lastPage' => $stories->lastPage(),
]
],
);
}

public function create(CreateInfluencerRequest $request, YoutubeScraper $youtube)
{
// Check ability
abort_if(Gate::denies('create_influencer') && !Auth()->user()->is_superadmin, Response::HTTP_FORBIDDEN, "403 Forbidden");
Expand Down
Loading

0 comments on commit 6f94e1e

Please sign in to comment.