From 4dd6171549ce1a9cadbc69a10a9264a67c79f0d0 Mon Sep 17 00:00:00 2001 From: benji7425 Date: Mon, 11 Dec 2017 01:05:18 +0000 Subject: [PATCH] Refactor for efficiency and improved readability --- app/index.js | 54 ++++++++++------ app/legacy-upgrader.js | 1 + app/models/feed-data.js | 95 ++++++++++++++++------------ app/models/guild-data.js | 21 ++++-- discord-bot-core/BaseEmbeddedData.js | 8 +++ discord-bot-core/index.js | 1 + package.json | 2 +- 7 files changed, 115 insertions(+), 67 deletions(-) create mode 100644 discord-bot-core/BaseEmbeddedData.js diff --git a/app/index.js b/app/index.js index 21a2779..91702fc 100644 --- a/app/index.js +++ b/app/index.js @@ -14,10 +14,8 @@ const guildsIterator = (function* () { } })(); -const token = require("../" + process.argv[2]).token, - dataFile = process.argv[3]; - -const client = new Core.Client(token, dataFile, __dirname + "/commands", GuildData); +// @ts-ignore +const client = new Core.Client(require("../token.json"), __dirname + "/commands", GuildData); client.on("beforeLogin", () => setInterval(doGuildIteration, Config.feedCheckInterval)); @@ -32,31 +30,45 @@ client.on("message", message => { return; client.guildDataModel.findOne({ guildID: message.guild.id }) - .then(guildData => { - if (guildData) { - guildData.feeds.forEach(feedData => - message.channel.id === feedData.channelID && feedData.cache(...GetUrls(message.content))); - guildData.save(); - } - }); + .then(guildData => guildData && cacheUrlsInMessage(message, guildData)); }); client.bootstrap(); //INTERNAL FUNCTIONS// -function doGuildIteration() { - const guild = guildsIterator.next().value; - guild && client.guildDataModel.findOne({ guildID: guild.id }) - .then(guildData => guildData && guildData.checkFeeds(guild).then(() => guildData.save())); -} - function parseLinksInGuilds() { const promises = []; + client.guildDataModel.find().then(guildDatas => - guildDatas.forEach(guildData => { - if (client.guilds.get(guildData.guildID)) - promises.push(guildData.cachePastPostedLinks(client.guilds.get(guildData.guildID))); - })); + guildDatas + .filter(guildData => client.guilds.get(guildData.guildID)) + .map(guildData => ({ guildData, guild: client.guilds.get(guildData.guildID) })) + .forEach(({ guildData, guild }) => promises.push(guildData.cachePastPostedLinks(guild).catch())) + ); return Promise.all(promises); +} + +function doGuildIteration() { + const guild = guildsIterator.next().value; + + if (guild) + client.guildDataModel.findOne({ guildID: guild.id }) + .then(guildData => guildData && checkGuildFeeds(guild, guildData)); +} + +function checkGuildFeeds(guild, guildData) { + guildData.checkFeeds(guild) + .then(values => values.some(x => x) && guildData.save()); +} + +function cacheUrlsInMessage(message, guildData) { + const anyNewLinksPosted = []; + + guildData.feeds + .filter(feedData => message.channel.id === feedData.channelID) + .forEach(feedData => anyNewLinksPosted.push(feedData.cache(...GetUrls(message.content)))); + + if (anyNewLinksPosted.some(x => x)) + guildData.save(); } \ No newline at end of file diff --git a/app/legacy-upgrader.js b/app/legacy-upgrader.js index 0b89945..a22728c 100644 --- a/app/legacy-upgrader.js +++ b/app/legacy-upgrader.js @@ -1,3 +1,4 @@ +// @ts-nocheck const NewGuildData = require("./models/guild-data.js"); const NewFeedData = require("./models/feed-data.js"); const FileSystem = require("fs"); diff --git a/app/models/feed-data.js b/app/models/feed-data.js index 243548b..d4bfac9 100644 --- a/app/models/feed-data.js +++ b/app/models/feed-data.js @@ -1,35 +1,49 @@ -const DiscordUtil = require("../../discord-bot-core").util; -const Camo = require("camo"); +// @ts-ignore const Config = require("../config.json"); -const Dns = require("dns"); //for host resolution checking -const Url = require("url"); //for url parsing -const { promisify } = require("util"); -const FeedReadPromise = promisify(require("feed-read")); //for extracing new links from RSS feeds -const DnsResolvePromise = promisify(Dns.resolve); -const GetUrls = require("get-urls"); //for extracting urls from messages -module.exports = class FeedData extends Camo.EmbeddedDocument { +const { promisify } = require("util"); +const Camo = require("camo"); +const Core = require("../../discord-bot-core"); +const DiscordUtil = require("../../discord-bot-core").util; +const GetUrls = require("get-urls"); +const Url = require("url"); + +// @ts-ignore +const readFeed = url => promisify(require("feed-read"))(url); +const resolveDns = promisify(require("dns").resolve); + +module.exports = class FeedData extends Core.BaseEmbeddedData { constructor() { super(); - this.feedID = String; - this.url = String; - this.channelID = String; - this.roleID = String; - this.cachedLinks = [String]; - this.maxCacheSize = Number; + this.feedID = ""; + this.url = ""; + this.channelID = ""; + this.roleID = ""; + this.cachedLinks = []; + this.maxCacheSize = 100; + + // @ts-ignore + this.schema({ + feedID: String, + url: String, + channelID: String, + roleID: String, + cachedLinks: [String], + maxCacheSize: Number + }); } cache(...elements) { - Array.prototype.push.apply( - this.cachedLinks, - elements - .map(el => normaliseUrl(el)) - .filter(el => !this.cachedLinks.includes(el)) - ); + const newArticles = elements + .map(el => normaliseUrl(el)) + .filter(el => this.cachedLinks.indexOf(el) === -1); - //seeing as new links come in at the end of the array, we need to remove the old links from the beginning - this.cachedLinks.splice(0, this.cachedLinks.length - this.maxCacheSize); + Array.prototype.push.apply(this.cachedLinks, newArticles); + + this.cachedLinks.splice(0, this.cachedLinks.length - this.maxCacheSize); //seeing as new links come in at the end of the array, we need to remove the old links from the beginning + + return elements.length > 0; } updatePastPostedLinks(guild) { @@ -51,9 +65,9 @@ module.exports = class FeedData extends Camo.EmbeddedDocument { } fetchLatest(guild) { - const dnsPromise = DnsResolvePromise(Url.parse(this.url).host).then(() => this._doFetchRSS(guild)); + const dnsPromise = resolveDns(Url.parse(this.url).host).then(() => this._doFetchRSS(guild)); - dnsPromise.catch(err => DiscordUtil.dateError("Connection error: Can't resolve host", err.message || err)); + dnsPromise.catch(err => DiscordUtil.dateDebugError("Connection error: Can't resolve host", err.message || err)); return dnsPromise; } @@ -64,7 +78,7 @@ module.exports = class FeedData extends Camo.EmbeddedDocument { } _doFetchRSS(guild) { - const feedPromise = FeedReadPromise(this.url).then(articles => this._processLatestArticle(guild, articles)); + const feedPromise = readFeed(this.url).then(articles => this._processLatestArticle(guild, articles)); feedPromise.catch(err => DiscordUtil.dateDebugError([`Error reading feed ${this.url}`, err])); @@ -73,31 +87,32 @@ module.exports = class FeedData extends Camo.EmbeddedDocument { _processLatestArticle(guild, articles) { if (articles.length === 0 || !articles[0].link) - return; - + return false; + const latest = normaliseUrl(articles[0].link); - - if (this.cachedLinks.includes(latest)) - return; - + + if (this.cachedLinks.indexOf(latest) > -1) + return false; + this.cache(latest); - + const channel = guild.channels.get(this.channelID), role = guild.roles.get(this.roleID); - + channel.send((role || "") + formatPost(articles[0])) .catch(err => DiscordUtil.dateDebugError(`Error posting in ${channel.id}: ${err.message || err}`)); + + return true; } }; function formatPost(article) { let message = ""; - if (article.title) - message += `\n**${article.title}**`; - if (article.content) - message += article.content.length > Config.charLimit ? "\nArticle content too long for a single Discord message!" : `\n${article.content}`; - if (article.link) - message += `\n\n${normaliseUrl(article.link)}`; + + if (article.title) message += `\n**${article.title}**`; + if (article.content) message += article.content.length > Config.charLimit ? "\nArticle content too long for a single Discord message!" : `\n${article.content}`; + if (article.link) message += `\n\n${normaliseUrl(article.link)}`; + return message; } diff --git a/app/models/guild-data.js b/app/models/guild-data.js index 1eb4e8d..e3255ce 100644 --- a/app/models/guild-data.js +++ b/app/models/guild-data.js @@ -5,19 +5,30 @@ module.exports = class GuildData extends Core.BaseGuildData { constructor() { super(); - this.feeds = [FeedData]; + this.feeds = []; + + this.schema({ + feeds: [FeedData] + }); } cachePastPostedLinks(guild) { return Promise.all( - this.feeds.map(feed => guild.channels.get(feed.channelID) ? feed.updatePastPostedLinks(guild) : Promise.resolve()) + this.feeds + .filter(feed => feedIsActive(feed, guild)) + .map(feed => feed.updatePastPostedLinks(guild)) ); } checkFeeds(guild) { return Promise.all( - this.feeds.map(feed => - guild.channels.get(feed.channelID) ? feed.fetchLatest(guild) : Promise.resolve()) + this.feeds + .filter(feed => feedIsActive(feed, guild)) + .map(feed => feed.fetchLatest(guild)) ); } -}; \ No newline at end of file +}; + +function feedIsActive(feed, guild) { + return guild.channels.get(feed.channelID); +} \ No newline at end of file diff --git a/discord-bot-core/BaseEmbeddedData.js b/discord-bot-core/BaseEmbeddedData.js new file mode 100644 index 0000000..43d4aa7 --- /dev/null +++ b/discord-bot-core/BaseEmbeddedData.js @@ -0,0 +1,8 @@ +const Camo = require("camo"); + +// @ts-ignore +module.exports = class BaseEmbeddedData extends Camo.EmbeddedDocument { + constructor() { + super(); + } +}; \ No newline at end of file diff --git a/discord-bot-core/index.js b/discord-bot-core/index.js index efaf166..532a802 100644 --- a/discord-bot-core/index.js +++ b/discord-bot-core/index.js @@ -4,6 +4,7 @@ const InternalConfig = require("./internal-config.json"); module.exports = { Client: require("./Client.js"), BaseGuildData: require("./BaseGuildData.js"), + BaseEmbeddedData: require("./BaseEmbeddedData.js"), Command: require("./Command.js"), util: require("./Util.js"), details: { diff --git a/package.json b/package.json index 551dfd8..bd5ff32 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "main": "app/index.js", "scripts": { "postinstall": "cd ./discord-bot-core && npm install", - "start": "node app/index.js token.json guilds.json --max-old-space-size=64 --name=rss-feed" + "start": "node app/index.js --max-old-space-size=64 --name=rss-feed" }, "dependencies": { "camo": "git+https://github.com/scottwrobinson/camo.git",