feat: add sitemap and robots (resolves #1388) (#1959)

* feat: add sitemap and robots

* fix: update return value of new console commands

* fix: update bsed on PR feedback

* fix: update defined terms test case

* fix: update based on PR feedback

* fix: update kernel command names

* fix: add proper lastmod for ToS and Privacy Policy

* fix: remove local environment for seo kernel commands

* fix: update based on PR feedback

* fix: update based on PR feedback

* fix: add 'en' locale to alternate link as well

.gitignore

@@ -1,6 +1,8 @@
 /node_modules
 /public/hot
 /public/storage
+/public/robots.txt
+/public/sitemap.xml
 /storage/*.key
 /vendor
 .DS_Store

app/Console/Commands/GenerateRobots.php

@@ -0,0 +1,32 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+
+class GenerateRobots extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'seo:generate-robots';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Generate the robots.txt file.';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        file_put_contents('./public/robots.txt', view('robots')->render());
+
+        return 0;
+    }
+}

app/Console/Commands/GenerateSitemap.php

@@ -0,0 +1,60 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Models\Page;
+use Carbon\Carbon;
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\Route;
+
+class GenerateSitemap extends Command
+{
+    /**
+     * The console command name.
+     *
+     * @var string
+     */
+    protected $signature = 'seo:generate-sitemap';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Generate the sitemap.';
+
+    /**
+     * Execute the console command.
+     *
+     * @return mixed
+     */
+    public function handle()
+    {
+        $routes = ['/' => ['en' => 'en', 'asl' => 'asl', 'fr' => 'fr', 'lsq' => 'lsq']];
+        // once deployed to the server, files have the same modified date, use README as a default last modified date
+        $lastmod = ['default' => Carbon::createFromTimestamp(filemtime('./README.md'))->toISOString()];
+        foreach (Route::getRoutes()->get('GET') as $route) {
+            if ($route->named(config('seo.sitemap.patterns'))) {
+                $routeURI = $route->uri();
+                [$locale, $url] = explode('/', $routeURI, 2);
+                if (array_key_exists($url, $routes)) {
+                    $routes[$url][$locale] = $routeURI;
+                } else {
+                    $routes[$url] = [$locale => $routeURI];
+                    if ($route->named(config('seo.sitemap.pages'))) {
+                        $routeName = explode('.', $route->getName());
+                        /*
+                         * TODO: come up with better query for the page, as slug may not follow the pattern in route name
+                         * or changes to the slug would break this logic: https://github.com/accessibility-exchange/platform/issues/1973
+                         */
+                        $page = Page::firstWhere('slug->en', '=', end($routeName));
+                        $lastmod[$url] = $page?->updated_at?->toISOString();
+                    }
+                }
+            }
+        }
+        file_put_contents('./public/sitemap.xml', view('sitemap', ['routes' => $routes, 'lastmod' => $lastmod])->render());
+
+        return 0;
+    }
+}

app/Console/Kernel.php

@@ -39,6 +39,16 @@ protected function schedule(Schedule $schedule)
             ->daily() // Run daily at midnight
             ->timezone('America/Los_Angeles') // Run as PST timezone
             ->onOneServer(); // run only on a single server at once
+
+        $schedule->command('seo:generate-sitemap') // generate sitemap
+            ->daily() // Run daily at midnight
+            ->environments(['staging', 'dev', 'production']) // only run for APP_ENV tagged staging, dev, local, or production
+            ->timezone('America/Los_Angeles'); // Run as PST timezone
+
+        $schedule->command('seo:generate-robots') // generate robots
+            ->daily() // Run daily at midnight
+            ->environments(['staging', 'dev', 'production']) // only run for APP_ENV tagged staging, dev, local, or production
+            ->timezone('America/Los_Angeles'); // Run as PST timezone
     }
 
     /**

config/seo.php

@@ -0,0 +1,21 @@
+<?php
+
+return [
+
+    /*
+    |--------------------------------------------------------------------------
+    | Search Engine Optimization
+    |--------------------------------------------------------------------------
+    |
+    | This file is for storing the configuration values related to search engine optimization.
+    |
+    */
+
+    'sitemap' => [
+        'patterns' => '*.about.*',
+        'pages' => [
+            '*.about.terms-of-service',
+            '*.about.privacy-policy',
+        ],
+    ],
+];

resources/views/robots.blade.php

@@ -0,0 +1,10 @@
+<?php header('Content-Type: text/plain; charset=utf-8'); ?>
+Sitemap: {{ env('APP_URL') }}/sitemap.xml
+
+User-Agent: *
+Disallow: admin/
+Disallow: api/
+Disallow: status/
+
+User-Agent: GPTBot
+Disallow: /

resources/views/sitemap.blade.php

@@ -0,0 +1,20 @@
+<?= '<' . '?' . 'xml version="1.0" encoding="UTF-8"?>' . "\n" ?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml"
+    xmlns:image="http://www.google.com/schemas/sitemap-image/1.1"
+    xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"
+    xmlns:news="http://www.google.com/schemas/sitemap-news/0.9">
+    @foreach ($routes as $baseUrl => $localizedUrls)
+        <url>
+            @if ($baseUrl !== 'about/page/{page}')
+                @foreach ($localizedUrls as $locale => $localizedUrl)
+                    @if ($locale === 'en')
+                        <loc>{{ env('APP_URL') . '/' . $localizedUrl }}</loc>
+                        <lastMod>{{ $lastmod[$baseUrl] ?? $lastmod['default'] }}</lastMod>
+                    @endif
+                    <link hreflang="{{ $locale }}" href="{{ env('APP_URL') . '/' . $localizedUrl }}"
+                        rel="alternate" />
+                @endforeach
+            @endif
+        </url>
+    @endforeach
+</urlset>

routes/defined-terms.php

@@ -4,4 +4,4 @@
 use Illuminate\Support\Facades\Route;
 
 Route::multilingual('/about/glossary', [DefinedTermController::class, 'index'])
-    ->name('defined-terms.index');
+    ->name('about.defined-terms.index');

tests/Feature/DefinedTermTest.php

@@ -7,6 +7,6 @@
     $user = User::factory()->create();
     $term = DefinedTerm::factory()->create();
 
-    $response = $this->actingAs($user)->get(localized_route('defined-terms.index'));
+    $response = $this->actingAs($user)->get(localized_route('about.defined-terms.index'));
     $response->assertSee($term->term);
 });