Added guid for tracking posts, added atom support

Author: ssddanbrown

.dockerignore

@@ -6,3 +6,4 @@
 /storage/database/*
 /storage/framework/*
 /storage/logs/*
+/.github

README.md

@@ -8,6 +8,7 @@ A simple, opinionated, RSS feed aggregator.
 
 The following features are built into the application:
 
+- Supports RSS and ATOM formats.
 - Regular auto-fetching of RSS feeds (Every hour).
 - Custom feed names and colors.
 - Feed-based tags for categorization.
@@ -30,6 +31,8 @@ This is not a list of planned features. Please see the [Low Maintenance Project]
 - No customization, extension or plugin system.
 - No organisation upon feed-level tagging.
 
+Upon the above, it's quite likely you'll come across issues. This project was created to meet a personal need while learning some new technologies. Much of the logic is custom written instead of using battle-tested libraries. 
+
 ## Screenshots
 
 TODO

app/Jobs/RefreshFeedJob.php

@@ -41,7 +41,7 @@ public function handle(FeedPostFetcher $postFetcher)
 
         foreach ($freshPosts as $post) {
             $post = $this->feed->posts()->updateOrCreate(
-                ['url' => $post->url],
+                ['guid' => $post->guid],
                 $post->getAttributes(),
             );
 

app/Models/Post.php

@@ -9,6 +9,6 @@ class Post extends Model
 {
     use HasFactory;
 
-    protected $fillable = ['url', 'title', 'description', 'published_at'];
-    protected $hidden = ['id', 'feed_id', 'created_at', 'updated_at'];
+    protected $fillable = ['url', 'title', 'description', 'published_at', 'guid'];
+    protected $hidden = ['id', 'feed_id', 'guid', 'created_at', 'updated_at'];
 }

app/Rss/FeedPostFetcher.php

@@ -24,7 +24,9 @@ public function fetchForFeed(Feed $feed): array
         }
 
         $rssData = ltrim($feedResponse->body());
-        if (substr($rssData, 0, 6) !== '<?xml ') {
+        $tagStart = explode(' ', substr($rssData, 0, 20))[0];
+        $validStarts = ['<?xml', '<feed', '<rss'];
+        if (!in_array($tagStart, $validStarts)) {
             return [];
         }
 

app/Rss/RssParser.php

@@ -28,21 +28,23 @@ public function rssDataToPosts(string $rssData): array
      */
     public function parseRssDataToPosts(string $rssData): array
     {
+        $rssData = trim($rssData);
+
         $rssXml = new SimpleXMLElement($rssData);
-        $items = iterator_to_array($rssXml->channel->item, false);
+        $items = is_iterable($rssXml->channel->item ?? null) ? iterator_to_array($rssXml->channel->item, false) : [];
+
+        $isAtom = false;
+        if (empty($items)) {
+            $items = is_iterable($rssXml->entry ?? null) ? iterator_to_array($rssXml->entry, false) : [];
+            $isAtom = true;
+        }
+
         $posts = [];
 
         foreach ($items as $item) {
+            $postData = $isAtom ? $this->getPostDataForAtomItem($item) : $this->getPostDataForRssItem($item);
 
-            $date = DateTime::createFromFormat('D, d M Y H:i:s T', $item->pubDate ?? '');
-            $postData = [
-                'title' => substr(strval($item->title ?? ''), 0, 250),
-                'description' => $this->formatDescription(strval($item->description) ?: ''),
-                'url' => strval($item->link ?? ''),
-                'published_at' => $date ? $date->getTimestamp() : 0,
-            ];
-
-            if (!$this->isValidRssData($postData)) {
+            if (!$this->isValidPostData($postData)) {
                 continue;
             }
 
@@ -52,23 +54,54 @@ public function parseRssDataToPosts(string $rssData): array
         return $posts;
     }
 
+    protected function getPostDataForRssItem(SimpleXMLElement $item): array
+    {
+        $date = DateTime::createFromFormat(DateTime::RSS, $item->pubDate ?? '');
+        $item = [
+            'title' => substr(strval($item->title ?? ''), 0, 250),
+            'description' => $this->formatDescription(strval($item->description) ?: ''),
+            'url' => strval($item->link ?? ''),
+            'guid' => strval($item->guid ?? ''),
+            'published_at' => $date ? $date->getTimestamp() : 0,
+        ];
+
+        if (empty($item['guid'])) {
+            $item['guid'] = $item['url'];
+        }
+
+        return $item;
+    }
+
     protected function formatDescription(string $description): string
     {
-        $decoded = html_entity_decode(strip_tags($description));
-        
+        $decoded = trim(html_entity_decode(strip_tags($description)));
+        $decoded = preg_replace('/\s+/', ' ', $decoded);
+
         if (strlen($decoded) > 200) {
             return substr($decoded, 0, 200) . '...';
         }
 
         return $decoded;
     }
 
+    protected function getPostDataForAtomItem(SimpleXMLElement $item): array
+    {
+        $date = new DateTime(strval($item->published ?? $item->updated ?? ''));
+        return [
+            'title' => html_entity_decode(substr(strval($item->title ?? ''), 0, 250)),
+            'description' => $this->formatDescription(strval($item->summary) ?: strval($item->content) ?: ''),
+            'url' => $item->link ? strval($item->link->attributes()['href']) : '',
+            'guid' => strval($item->id ?? ''),
+            'published_at' => $date ? $date->getTimestamp() : 0,
+        ];
+    }
+
     /**
      * @param array{title: string, description: string, url: string, published_at: int} $item
      */
-    protected function isValidRssData(array $item): bool
+    protected function isValidPostData(array $item): bool
     {
-        if (empty($item['title']) || empty($item['url'])) {
+        if (empty($item['title']) || empty($item['url']) || empty($item['guid'])) {
             return false;
         }
 

config/queue.php

@@ -86,7 +86,7 @@
 
     'failed' => [
         'driver' => env('QUEUE_FAILED_DRIVER', 'database-uuids'),
-        'database' => env('DB_CONNECTION', 'mysql'),
+        'database' => env('DB_CONNECTION', 'sqlite'),
         'table' => 'failed_jobs',
     ],
 

database/factories/PostFactory.php

@@ -17,12 +17,14 @@ class PostFactory extends Factory
      */
     public function definition()
     {
+        $url = $this->faker->url . '?query=' . random_int(0, 1000);
         return [
             'feed_id' => Feed::factory(),
             'published_at' => now()->subHours(random_int(0, 200))->unix(),
             'title' => $this->faker->title,
             'description' => $this->faker->words(50, true),
-            'url' => $this->faker->url . '?query=' . random_int(0, 1000),
+            'url' => $url,
+            'guid' => $url,
             'thumbnail' => ''
         ];
     }

database/migrations/2022_06_29_124610_create_posts_table.php

@@ -20,10 +20,11 @@ public function up()
             $table->string('title', 250);
             $table->text('description');
             $table->string('url', 250);
+            $table->string('guid', 250);
             $table->string('thumbnail')->default('');
             $table->timestamps();
 
-            $table->unique(['feed_id', 'url']);
+            $table->unique(['feed_id', 'guid']);
         });
     }
 

tests/Unit/RssParserTest.php

@@ -43,10 +43,11 @@ public function test_it_parses_valid_posts()
   </channel>
 </rss>
 END
-);
+        );
 
         $this->assertCount(2, $posts);
         $this->assertEquals('BookStack Release v22.06', $posts[0]->title);
+        $this->assertEquals('https://www.bookstackapp.com/blog/bookstack-release-v22-06/', $posts[0]->guid);
         $this->assertEquals('https://www.bookstackapp.com/blog/bookstack-release-v22-06/', $posts[0]->url);
         $this->assertEquals(1656068400, $posts[0]->published_at);
         $this->assertEquals('BookStack v22.06 is now here! This release was primarily refinement focused but it does include some great new features that may streamline your usage of the platform.', $posts[0]->description);
@@ -75,6 +76,7 @@ public function test_it_parses_single_post()
 
         $this->assertCount(1, $posts);
         $this->assertEquals('BookStack Release v22.06', $posts[0]->title);
+        $this->assertEquals('https://www.bookstackapp.com/blog/bookstack-release-v22-06/', $posts[0]->guid);
         $this->assertEquals('https://www.bookstackapp.com/blog/bookstack-release-v22-06/', $posts[0]->url);
         $this->assertEquals(1656068400, $posts[0]->published_at);
         $this->assertEquals('BookStack v22.06 is now here! This release was primarily refinement focused but it does include some great new features that may streamline your usage of the platform.', $posts[0]->description);
@@ -156,4 +158,106 @@ public function test_descriptions_in_html_are_parsed()
         $this->assertEquals('Some really cool text & with £ entities within', $posts[0]->description);
     }
 
+    public function test_it_parses_valid_atom_feeds()
+    {
+        $parser = new RssParser();
+
+        $posts = $parser->rssDataToPosts(<<<END
+<?xml version="1.0" encoding="utf-8" standalone="yes"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <title type="text" xml:lang="en">Example Atom Feed</title>
+
+  <entry>
+    <title>Example Post A</title>
+    <link href="https://example.com/a"/>
+    <updated>2022-06-09T17:00:00.000Z</updated>
+    <id>https://example.com/a</id>
+    <content type="html">
+      &lt;p&gt;Example Post A&lt;/p&gt;
+      &lt;p&gt;&lt;a href="https://example/a"&gt;Read the full article&lt;/a&gt;&lt;/p&gt;
+    </content>
+    <author>
+        <name>Example Team</name>
+    </author>
+  </entry>
+
+  <entry>
+    <title>Example Post B</title>
+    <link href="https://example.com/b"/>
+    <updated>2022-06-08T17:00:00.000Z</updated>
+    <id>https://example.com/b</id>
+    <content type="html">
+      &lt;p&gt;Example Post B&lt;/p&gt;
+      &lt;p&gt;&lt;a href="https://example/a"&gt;Read the full article&lt;/a&gt;&lt;/p&gt;
+    </content>
+    <author>
+        <name>Example Team</name>
+    </author>
+  </entry>
+</feed>
+END
+        );
+
+        $this->assertCount(2, $posts);
+        $this->assertEquals('Example Post A', $posts[0]->title);
+        $this->assertEquals('https://example.com/a', $posts[0]->guid);
+        $this->assertEquals('https://example.com/a', $posts[0]->url);
+        $this->assertEquals(1654794000, $posts[0]->published_at);
+        $this->assertEquals("Example Post A Read the full article", $posts[0]->description);
+    }
+
+    public function test_atom_summary_used_over_content()
+    {
+        $parser = new RssParser();
+
+        $posts = $parser->rssDataToPosts(<<<END
+<?xml version="1.0" encoding="utf-8" standalone="yes"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <title type="text" xml:lang="en">Example Atom Feed</title>
+
+  <entry>
+    <title>Example Post A</title>
+    <link href="https://example.com/a"/>
+    <updated>2022-06-09T17:00:00.000Z</updated>
+    <id>https://example.com/a</id>
+    <content type="html">&lt;p&gt;Example Post A Content&lt;/p&gt;</content>
+    <summary type="html">&lt;p&gt;Example Post A Summary&lt;/p&gt;</summary>
+    <author>
+        <name>Example Team</name>
+    </author>
+  </entry>
+</feed>
+END
+        );
+
+        $this->assertEquals("Example Post A Summary", $posts[0]->description);
+    }
+
+    public function test_switcher_summary_used_over_content()
+    {
+        $parser = new RssParser();
+
+        $posts = $parser->rssDataToPosts(<<<END
+<?xml version="1.0" encoding="utf-8" standalone="yes"?>
+<feed xmlns="http://www.w3.org/2005/Atom">
+  <title type="text" xml:lang="en">Example Atom Feed</title>
+
+  <entry>
+    <title>Example Post A</title>
+    <link href="https://example.com/a"/>
+    <updated>2022-06-09T17:00:00.000Z</updated>
+    <id>https://example.com/a</id>
+    <content type="html">&lt;p&gt;Example Post A Content&lt;/p&gt;</content>
+    <summary type="html">&lt;p&gt;Example Post A Summary&lt;/p&gt;</summary>
+    <author>
+        <name>Example Team</name>
+    </author>
+  </entry>
+</feed>
+END
+        );
+
+        $this->assertEquals("Example Post A Summary", $posts[0]->description);
+    }
+
 }