From f02602955a6bc15f34a4fec9f0221842ef0f4b02 Mon Sep 17 00:00:00 2001 From: benji7425 Date: Sat, 12 Aug 2017 01:55:58 +0100 Subject: [PATCH 1/5] Add feed checking logic, set to run at 30 sec intervals --- app/config.json | 1 + app/index.js | 14 +++++++++--- app/models/feed-data.js | 46 ++++++++++++++++++++++++++++++++++++++++ app/models/guild-data.js | 6 +++++- 4 files changed, 63 insertions(+), 4 deletions(-) diff --git a/app/config.json b/app/config.json index 26971c6..ff568bc 100644 --- a/app/config.json +++ b/app/config.json @@ -1,5 +1,6 @@ { "saveIntervalSec": 60, + "feedCheckIntervalSec": 30, "commands": { "version": "version", "addFeed": "add-feed" diff --git a/app/index.js b/app/index.js index 68168c4..b02d198 100644 --- a/app/index.js +++ b/app/index.js @@ -20,8 +20,13 @@ module.exports = (client) => { const guildsData = FileSystem.existsSync(SAVE_FILE) ? fromJSON(JsonFile.readFileSync(SAVE_FILE)) : {}; setInterval(() => writeFile(guildsData), config.saveIntervalSec * 1000); - parseLinksInAllGuilds(client.guilds, guildsData).then(writeFile(guildsData)); + parseLinksInGuilds(client.guilds, guildsData).then(writeFile(guildsData)); + //set up an interval to check all the feeds + checkFeedsInGuilds(guildsData); + setInterval(() => checkFeedsInGuilds(guildsData), config.feedCheckIntervalSec * 1000); + + //set up an on message handler to detect when links are posted client.on("message", message => { if (message.author.id !== client.user.id) { //check the bot isn't triggering itself if (message.channel.type === "dm") @@ -69,11 +74,14 @@ function addFeed(client, guildsData, message) { } else responseMessage.reply("Your feed has not been saved, please add it again with the correct details"); - }); } -function parseLinksInAllGuilds(guilds, guildsData) { +function checkFeedsInGuilds(guildsData) { + guildsData.forEach(guild => guild.checkFeeds()); +} + +function parseLinksInGuilds(guilds, guildsData) { const promises = []; for (let guild of guilds) { const guildData = guildsData[guild.id]; diff --git a/app/models/feed-data.js b/app/models/feed-data.js index f4aae1c..7f47450 100644 --- a/app/models/feed-data.js +++ b/app/models/feed-data.js @@ -1,3 +1,11 @@ +//my imports +const DiscordUtil = require("discordjs-util"); + +//external lib imports +const Dns = require("dns"); +const Url = require("url"); +const FeedRead = require("feed-read"); + module.exports = class FeedData { constructor({ link, channelName, roleName, cachedLinks }) { this.link = link; @@ -23,8 +31,46 @@ module.exports = class FeedData { .catch(reject); }); } + + check(guild) { + Dns.resolve(Url.parse(this.link).host, err => { //check we can resolve the host, so we can throw an appropriate error if it fails + if (err) + DiscordUtil.dateError("Connection Error: Can't resolve host", err); //log our error if we can't resolve the host + else + FeedRead(this.link, (err, articles) => { //check the feed + if (err) + DiscordUtil.dateError(err); + else { + let latest = articles[0]; //extract the latest link + latest = normaliseUrl(latest); //standardise it a bit + + //if we don't have it cached already, cache it and callback + if (!this.cachedLinks.includes(latest)) { + this.cachedLinks.push(latest); + post(guild, latest); + } + } + }); + }); + } }; +function post(guild, link){ + const channel = guild.channels.first(ch => ch.type === "text" && ch.name.toLower() === this.channelName.toLower()); + channel.send(link); +} + +function normaliseUrl(url) { + url = url.replace("https://", "http://"); //cheaty way to get around http and https not matching + + if (Url.parse(url).host.includes("youtu")) //detect youtu.be and youtube.com - yes I know it's hacky + url = url.split("&")[0]; //quick way to chop off stuff like &feature=youtube + + url = url.replace("http://www.youtube.com/watch?v=", "http://youtu.be/"); //turn full url into share url + + return url; +} + function getUrls(str) { return str.match(/(\b(https?|ftp|file):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])/ig); } \ No newline at end of file diff --git a/app/models/guild-data.js b/app/models/guild-data.js index 82de0f9..1264809 100644 --- a/app/models/guild-data.js +++ b/app/models/guild-data.js @@ -9,11 +9,15 @@ module.exports = class GuildData { cachePastPostedLinks() { const promises = []; - + this.feeds.forEach(feed => { promises.push(feed.cachePastPostedLinks(this).catch(Util.dateError)); }); return Promise.all(promises); } + + checkFeeds() { + this.feeds.forEach(feed => feed.check()); + } }; \ No newline at end of file From 385f0f86561a1201623549349eda1cf43a5d4f2c Mon Sep 17 00:00:00 2001 From: benji7425 Date: Sat, 12 Aug 2017 02:05:59 +0100 Subject: [PATCH 2/5] Update formatting --- app/index.js | 42 +++++++++++------------------------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/app/index.js b/app/index.js index b02d198..6b0e71f 100644 --- a/app/index.js +++ b/app/index.js @@ -61,7 +61,12 @@ const HandleMessage = { }; function addFeed(client, guildsData, message) { - const feedData = createNewFeed(message); //create a new feed data instance from the data in our message + const parameters = message.content.split(" "); //expect !addfeed + const feedData = new FeedData({ + link: parameters[1], + channelName: parameters[2], + roleName: parameters[3] + }); //ask the user if they're happy with the details they set up, save if yes, don't if no DiscordUtil.ask(client, message.channel, message.member, "Are you happy with this? " + feedData) @@ -69,7 +74,10 @@ function addFeed(client, guildsData, message) { //if they responded yes, save the feed and let them know, else tell them to start again if (message.content.toLowerCase() === "yes") { - saveFeed(guildsData, message.guild.id, feedData); + if (!guildsData[message.guild.id]) + guildsData[message.guild.id] = new GuildData({ id: message.guild.id, feeds: [] }); + + guildsData[message.guild.id].feeds.push(feedData); responseMessage.reply("Your new feed has been saved!"); } else @@ -78,7 +86,7 @@ function addFeed(client, guildsData, message) { } function checkFeedsInGuilds(guildsData) { - guildsData.forEach(guild => guild.checkFeeds()); + Object.keys(guildsData).forEach(key => guildsData[key].checkFeeds()); } function parseLinksInGuilds(guilds, guildsData) { @@ -91,34 +99,6 @@ function parseLinksInGuilds(guilds, guildsData) { return Promise.all(promises); } -/** - * Create a new feed from the message object where the user is setting it up - * @param {Discord.Message} message - * @returns {FeedData} Newly created feed data object - */ -function createNewFeed(message) { - const parameters = message.content.split(" "); //expect !addfeed - const feedData = new FeedData({ - link: parameters[1], - channelName: parameters[2], - roleName: parameters[3] - }); - return feedData; -} - -/** - * Saves a passed feed data object into the passed guildsData object, for the specified guild - * @param {object} guildsData - * @param {string} guildID - * @param {FeedData} feedData - */ -function saveFeed(guildsData, guildID, feedData) { - if (!guildsData[guildID]) - guildsData[guildID] = new GuildData({ id: guildID, feeds: [] }); - - guildsData[guildID].feeds.push(feedData); -} - function writeFile(guildsData) { JsonFile.writeFile(SAVE_FILE, guildsData, err => { if (err) DiscordUtil.dateError("Error writing file", err); }); } From ac9da5f8797062ce94f161c402dea351c3fa0b27 Mon Sep 17 00:00:00 2001 From: benji7425 Date: Sun, 13 Aug 2017 03:12:20 +0100 Subject: [PATCH 3/5] Fix issues with feed addition --- app/index.js | 24 +++++++++++++++++------- app/models/feed-data.js | 12 ++++++------ app/models/guild-data.js | 2 +- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/app/index.js b/app/index.js index 6b0e71f..9f9a737 100644 --- a/app/index.js +++ b/app/index.js @@ -62,22 +62,31 @@ const HandleMessage = { function addFeed(client, guildsData, message) { const parameters = message.content.split(" "); //expect !addfeed + + const feedUrl = parameters[2], channelName = message.mentions.channels.first().name, roleName = parameters[4]; + + if (!feedUrl || !channelName) { + message.reply("Please supply all the needed fields in this format:\n add-feed url channel-name role-name"); + return; + } + const feedData = new FeedData({ - link: parameters[1], - channelName: parameters[2], - roleName: parameters[3] + url: feedUrl, + channelName: channelName, + roleName: roleName }); //ask the user if they're happy with the details they set up, save if yes, don't if no - DiscordUtil.ask(client, message.channel, message.member, "Are you happy with this? " + feedData) + DiscordUtil.ask(client, message.channel, message.member, "Are you happy with this?\n ```JavaScript\n" + JSON.stringify(feedData, null, "\n") + "```") .then(responseMessage => { //if they responded yes, save the feed and let them know, else tell them to start again - if (message.content.toLowerCase() === "yes") { + if (responseMessage.content.toLowerCase() === "yes") { if (!guildsData[message.guild.id]) guildsData[message.guild.id] = new GuildData({ id: message.guild.id, feeds: [] }); guildsData[message.guild.id].feeds.push(feedData); + writeFile(guildsData); responseMessage.reply("Your new feed has been saved!"); } else @@ -104,6 +113,7 @@ function writeFile(guildsData) { } function fromJSON(json) { - const guildIDs = Object.keys(json); - guildIDs.forEach(guildID => { guildIDs[guildID] = new GuildData(guildIDs[guildID]); }); + const guildsData = Object.keys(json); + guildsData.forEach(guildID => { json[guildID] = new GuildData(json[guildID]); }); + return json; } \ No newline at end of file diff --git a/app/models/feed-data.js b/app/models/feed-data.js index 7f47450..0a72d18 100644 --- a/app/models/feed-data.js +++ b/app/models/feed-data.js @@ -7,8 +7,8 @@ const Url = require("url"); const FeedRead = require("feed-read"); module.exports = class FeedData { - constructor({ link, channelName, roleName, cachedLinks }) { - this.link = link; + constructor({ url, channelName, roleName, cachedLinks }) { + this.url = url; this.channelName = channelName; this.roleName = roleName; this.cachedLinks = cachedLinks | []; @@ -33,11 +33,11 @@ module.exports = class FeedData { } check(guild) { - Dns.resolve(Url.parse(this.link).host, err => { //check we can resolve the host, so we can throw an appropriate error if it fails + Dns.resolve(Url.parse(this.url).host, err => { //check we can resolve the host, so we can throw an appropriate error if it fails if (err) DiscordUtil.dateError("Connection Error: Can't resolve host", err); //log our error if we can't resolve the host else - FeedRead(this.link, (err, articles) => { //check the feed + FeedRead(this.url, (err, articles) => { //check the feed if (err) DiscordUtil.dateError(err); else { @@ -55,9 +55,9 @@ module.exports = class FeedData { } }; -function post(guild, link){ +function post(guild, url){ const channel = guild.channels.first(ch => ch.type === "text" && ch.name.toLower() === this.channelName.toLower()); - channel.send(link); + channel.send(url); } function normaliseUrl(url) { diff --git a/app/models/guild-data.js b/app/models/guild-data.js index 1264809..65936ba 100644 --- a/app/models/guild-data.js +++ b/app/models/guild-data.js @@ -18,6 +18,6 @@ module.exports = class GuildData { } checkFeeds() { - this.feeds.forEach(feed => feed.check()); + // this.feeds.forEach(feed => feed.check()); } }; \ No newline at end of file From fbab7730b818357670849230d708d3910c2f8499 Mon Sep 17 00:00:00 2001 From: benji7425 Date: Sun, 13 Aug 2017 04:15:06 +0100 Subject: [PATCH 4/5] Fix a few more issues --- app/index.js | 24 +++++++++++++++--------- app/models/feed-data.js | 16 +++++++--------- app/models/guild-data.js | 6 +++--- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/app/index.js b/app/index.js index 9f9a737..e8f9d7e 100644 --- a/app/index.js +++ b/app/index.js @@ -3,6 +3,7 @@ const FileSystem = require("fs"); //external lib imports const JsonFile = require("jsonfile"); +const Url = require("url"); //my imports const DiscordUtil = require("discordjs-util"); @@ -23,8 +24,8 @@ module.exports = (client) => { parseLinksInGuilds(client.guilds, guildsData).then(writeFile(guildsData)); //set up an interval to check all the feeds - checkFeedsInGuilds(guildsData); - setInterval(() => checkFeedsInGuilds(guildsData), config.feedCheckIntervalSec * 1000); + checkFeedsInGuilds(client.guilds, guildsData); + setInterval(() => checkFeedsInGuilds(client.guilds, guildsData), config.feedCheckIntervalSec * 1000); //set up an on message handler to detect when links are posted client.on("message", message => { @@ -63,12 +64,17 @@ const HandleMessage = { function addFeed(client, guildsData, message) { const parameters = message.content.split(" "); //expect !addfeed - const feedUrl = parameters[2], channelName = message.mentions.channels.first().name, roleName = parameters[4]; + const channel = message.mentions.channels.first(); + if(!channel) + return message.reply("Please tag a channel with #channel-name"); - if (!feedUrl || !channelName) { - message.reply("Please supply all the needed fields in this format:\n add-feed url channel-name role-name"); - return; - } + const feedUrl = parameters[2], channelName = channel.name, roleName = parameters[4]; + + if (!Url.parse(feedUrl).host) + return message.reply("Please supply a valid url"); + + if (!feedUrl || !channelName) + return message.reply("Please supply all the needed fields in this format:\n add-feed url channel-name role-name"); const feedData = new FeedData({ url: feedUrl, @@ -94,8 +100,8 @@ function addFeed(client, guildsData, message) { }); } -function checkFeedsInGuilds(guildsData) { - Object.keys(guildsData).forEach(key => guildsData[key].checkFeeds()); +function checkFeedsInGuilds(guilds, guildsData) { + Object.keys(guildsData).forEach(key => guildsData[key].checkFeeds(guilds)); } function parseLinksInGuilds(guilds, guildsData) { diff --git a/app/models/feed-data.js b/app/models/feed-data.js index 0a72d18..3e3e954 100644 --- a/app/models/feed-data.js +++ b/app/models/feed-data.js @@ -11,7 +11,7 @@ module.exports = class FeedData { this.url = url; this.channelName = channelName; this.roleName = roleName; - this.cachedLinks = cachedLinks | []; + this.cachedLinks = cachedLinks || []; } /** @@ -33,7 +33,7 @@ module.exports = class FeedData { } check(guild) { - Dns.resolve(Url.parse(this.url).host, err => { //check we can resolve the host, so we can throw an appropriate error if it fails + Dns.resolve(Url.parse(this.url).host || "", err => { //check we can resolve the host, so we can throw an appropriate error if it fails if (err) DiscordUtil.dateError("Connection Error: Can't resolve host", err); //log our error if we can't resolve the host else @@ -41,13 +41,16 @@ module.exports = class FeedData { if (err) DiscordUtil.dateError(err); else { - let latest = articles[0]; //extract the latest link + let latest = articles[0].link; //extract the latest link latest = normaliseUrl(latest); //standardise it a bit //if we don't have it cached already, cache it and callback if (!this.cachedLinks.includes(latest)) { this.cachedLinks.push(latest); - post(guild, latest); + + const channel = guild.channels.find(ch => ch.type === "text" && ch.name.toLowerCase() === this.channelName.toLowerCase()); + const role = guild.roles.find(role => role.name.toLowerCase() === this.roleName.toLowerCase()); + channel.send(role + " " + latest); } } }); @@ -55,11 +58,6 @@ module.exports = class FeedData { } }; -function post(guild, url){ - const channel = guild.channels.first(ch => ch.type === "text" && ch.name.toLower() === this.channelName.toLower()); - channel.send(url); -} - function normaliseUrl(url) { url = url.replace("https://", "http://"); //cheaty way to get around http and https not matching diff --git a/app/models/guild-data.js b/app/models/guild-data.js index 65936ba..94253d2 100644 --- a/app/models/guild-data.js +++ b/app/models/guild-data.js @@ -4,7 +4,7 @@ const Util = require("discordjs-util"); module.exports = class GuildData { constructor({ id, feeds }) { this.id = id; - this.feeds = feeds.filter(feed => new FeedData(feed)); + this.feeds = feeds.map(feed => new FeedData(feed)); } cachePastPostedLinks() { @@ -17,7 +17,7 @@ module.exports = class GuildData { return Promise.all(promises); } - checkFeeds() { - // this.feeds.forEach(feed => feed.check()); + checkFeeds(guilds) { + this.feeds.forEach(feed => feed.check(guilds.get(this.id))); } }; \ No newline at end of file From 1539926671b143300a69f871e7c29fd4f31cd33a Mon Sep 17 00:00:00 2001 From: benji7425 Date: Tue, 22 Aug 2017 23:02:16 +0100 Subject: [PATCH 5/5] Update feed addition to be simpler --- app/index.js | 21 +++++++++------------ app/models/feed-data.js | 6 +++--- package.json | 10 +++++----- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/app/index.js b/app/index.js index e8f9d7e..e20e4ea 100644 --- a/app/index.js +++ b/app/index.js @@ -2,8 +2,9 @@ const FileSystem = require("fs"); //external lib imports -const JsonFile = require("jsonfile"); -const Url = require("url"); +const JsonFile = require("jsonfile"); //for saving to/from JSON +const Url = require("url"); //for url parsing +const GetUrls = require("get-urls"); //for extracting urls from messages //my imports const DiscordUtil = require("discordjs-util"); @@ -64,22 +65,18 @@ const HandleMessage = { function addFeed(client, guildsData, message) { const parameters = message.content.split(" "); //expect !addfeed + const feedUrl = [...GetUrls(message.content)][0]; const channel = message.mentions.channels.first(); - if(!channel) - return message.reply("Please tag a channel with #channel-name"); - const feedUrl = parameters[2], channelName = channel.name, roleName = parameters[4]; + if (!feedUrl || !channel) + return message.reply("Please provide both a channel and an RSS feed URL. You can optionally @mention a role also."); - if (!Url.parse(feedUrl).host) - return message.reply("Please supply a valid url"); - - if (!feedUrl || !channelName) - return message.reply("Please supply all the needed fields in this format:\n add-feed url channel-name role-name"); + const role = message.mentions.roles.first(); const feedData = new FeedData({ url: feedUrl, - channelName: channelName, - roleName: roleName + channelName: channel.name, + roleName: role.name }); //ask the user if they're happy with the details they set up, save if yes, don't if no diff --git a/app/models/feed-data.js b/app/models/feed-data.js index 3e3e954..7ff2e80 100644 --- a/app/models/feed-data.js +++ b/app/models/feed-data.js @@ -2,9 +2,9 @@ const DiscordUtil = require("discordjs-util"); //external lib imports -const Dns = require("dns"); -const Url = require("url"); -const FeedRead = require("feed-read"); +const Dns = require("dns"); //for host resolution checking +const Url = require("url"); //for url parsing +const FeedRead = require("feed-read"); //for extracing new links from RSS feeds module.exports = class FeedData { constructor({ url, channelName, roleName, cachedLinks }) { diff --git a/package.json b/package.json index b918016..0f2b9b7 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,8 @@ "description": "", "main": "app/index.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1", - "start": "node wrapper.js" + "test": "echo \"Error: no test specified\" && exit 1", + "start": "node wrapper.js" }, "repository": { "type": "git", @@ -22,7 +22,7 @@ "discordjs-util": "git+https://github.com/benji7425/discordjs-util.git", "dns": "0.2.2", "feed-read": "0.0.1", - "jsonfile": "3.0.1", - "urijs": "1.18.10" + "get-urls": "7.0.0", + "jsonfile": "3.0.1" } -} \ No newline at end of file +}